meerschaum 2.7.6__py3-none-any.whl → 2.7.7__py3-none-any.whl

meerschaum/actions/drop.py

@@ -6,52 +6,65 @@ Functions for dropping elements
 """
 
 from __future__ import annotations
-from meerschaum.utils.typing import SuccessTuple, Union, Any, Optional, Sequence, List
+from meerschaum.utils.typing import SuccessTuple, Any, Optional, List
+
 
 def drop(
-        action: Optional[List[str]] = None,
-        **kw: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    **kw: Any
+) -> SuccessTuple:
     """
     Drop pipe data (maintaining registration) or tables.
     """
     from meerschaum.actions import choose_subaction
     options = {
         'pipes'  : _drop_pipes,
+        'indices': _drop_indices,
+        'index': _drop_indices,
+        'indexes': _drop_indices,
     }
     return choose_subaction(action, options, **kw)
 
 
 def _drop_pipes(
-        action: Optional[List[str]] = None,
-        yes: bool = False,
-        force: bool = False,
-        noask: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    yes: bool = False,
+    force: bool = False,
+    noask: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """
     Drop pipes' tables but keep pipe metadata registration.
     """
-    from meerschaum.utils.prompt import prompt, yes_no
+    from meerschaum.utils.prompt import yes_no
     from meerschaum import get_pipes
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.debug import dprint
     from meerschaum.utils.formatting import pprint
+
     pipes = get_pipes(as_list=True, debug=debug, **kw)
     if len(pipes) == 0:
         return False, "No pipes to drop."
+
     question = (
         "Are you sure you want to drop these tables?\n    "
-        + "Data will be lost and will need to be resynced.\n\n"
+        "Data will be lost and will need to be resynced.\n\n"
     )
-    for p in pipes:
-        question += f"    - {p.target}" + "\n"
+    seen_targets = set()
+    for pipe in pipes:
+        target = pipe.target
+        if target in seen_targets:
+            continue
+        question += f"    - {target}" + "\n"
+        seen_targets.add(target)
+
     question += '\n'
     if force:
         answer = True
     else:
         answer = yes_no(question, default='n', noask=noask, yes=yes)
+
     if not answer:
         return False, "No pipes were dropped."
 
@@ -59,22 +72,87 @@ def _drop_pipes(
     successes, fails = 0, 0
     msg = ""
 
-    for p in pipes:
-        success_tup = p.drop(debug=debug)
-        success_dict[p] = success_tup[1]
-        if success_tup[0]:
+    for pipe in pipes:
+        drop_success, drop_msg = pipe.drop(debug=debug)
+        success_dict[pipe] = drop_msg
+        if drop_success:
             successes += 1
         else:
             fails += 1
-            warn(success_tup[1], stack=False)
+            warn(drop_msg, stack=False)
     
     if debug:
         dprint("Results for dropping pipes.")
         pprint(success_dict)
 
     msg = (
-        f"Finished dropping {len(pipes)} pipes\n" + 
-        f"    ({successes} succeeded, {fails} failed)."
+        f"Finished dropping {len(pipes)} pipes"
+        + ('s' if len(pipes) != 1 else '')
+        + f"\n    ({successes} succeeded, {fails} failed)."
+    )
+    return successes > 0, msg
+
+
+def _drop_indices(
+    action: Optional[List[str]] = None,
+    yes: bool = False,
+    force: bool = False,
+    noask: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
+    """
+    Drop pipes' tables but keep pipe metadata registration.
+    """
+    from meerschaum.utils.prompt import yes_no
+    from meerschaum import get_pipes
+    from meerschaum.utils.warnings import warn
+    from meerschaum.utils.debug import dprint
+    from meerschaum.utils.formatting import pprint
+
+    pipes = get_pipes(as_list=True, debug=debug, **kw)
+    if len(pipes) == 0:
+        return False, "No pipes to drop."
+
+    question = "Are you sure you want to drop these indices?\n"
+    for pipe in pipes:
+        indices = pipe.get_indices()
+        if not indices:
+            continue
+        question += f"\n{pipe}\n"
+        for ix_key, index_name in indices.items():
+            question += f"    - {index_name}\n"
+
+    question += '\n'
+    if force:
+        answer = True
+    else:
+        answer = yes_no(question, default='n', noask=noask, yes=yes)
+
+    if not answer:
+        return False, "No pipes were dropped."
+
+    success_dict = {}
+    successes, fails = 0, 0
+    msg = ""
+
+    for pipe in pipes:
+        drop_success, drop_msg = pipe.drop_indices(debug=debug)
+        success_dict[pipe] = drop_msg
+        if drop_success:
+            successes += 1
+        else:
+            fails += 1
+            warn(drop_msg, stack=False)
+    
+    if debug:
+        dprint("Results for dropping indices.")
+        pprint(success_dict)
+
+    msg = (
+        f"Finished dropping indices for {len(pipes)} pipe"
+        + ('s' if len(pipes) != 1 else '')
+        + f"\n    ({successes} succeeded, {fails} failed)."
     )
     return successes > 0, msg
 

meerschaum/actions/index.py

@@ -0,0 +1,71 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+# vim:fenc=utf-8
+
+"""
+Functions for indexing tables.
+"""
+
+from __future__ import annotations
+from meerschaum.utils.typing import SuccessTuple, Any, Optional, List
+
+
+def index(
+    action: Optional[List[str]] = None,
+    **kw: Any
+) -> SuccessTuple:
+    """
+    Create pipes' indices.
+    """
+    from meerschaum.actions import choose_subaction
+    options = {
+        'pipes'  : _index_pipes,
+    }
+    return choose_subaction(action, options, **kw)
+
+
+def _index_pipes(
+    action: Optional[List[str]] = None,
+    yes: bool = False,
+    force: bool = False,
+    noask: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
+    """
+    Create pipes' indices.
+    """
+    from meerschaum import get_pipes
+    from meerschaum.utils.warnings import warn, info
+
+    pipes = get_pipes(as_list=True, debug=debug, **kw)
+    if len(pipes) == 0:
+        return False, "No pipes to index."
+
+    success_dict = {}
+    successes, fails = 0, 0
+    msg = ""
+
+    for pipe in pipes:
+        info(f"Creating indices for {pipe}...")
+        index_success, index_msg = pipe.create_indices(debug=debug)
+        success_dict[pipe] = index_msg
+        if index_success:
+            successes += 1
+        else:
+            fails += 1
+            warn(index_msg, stack=False)
+    
+    msg = (
+        f"Finished indexing {len(pipes)} pipes"
+        + ('s' if len(pipes) != 1 else '')
+        + f"\n    ({successes} succeeded, {fails} failed)."
+    )
+    return successes > 0, msg
+
+
+### NOTE: This must be the final statement of the module.
+###       Any subactions added below these lines will not
+###       be added to the `help` docstring.
+from meerschaum.actions import choices_docstring as _choices_docstring
+index.__doc__ += _choices_docstring('index')

meerschaum/actions/register.py

@@ -10,6 +10,7 @@ from __future__ import annotations
 import meerschaum as mrsm
 from meerschaum.utils.typing import SuccessTuple, Any, List, Optional, Dict
 
+
 def register(
     action: Optional[List[str]] = None,
     **kw: Any
@@ -70,10 +71,9 @@ def _register_pipes(
         `connector_keys` and `metric_keys`.
         If `location_keys` is empty, assume [`None`].
     """
-    from meerschaum import get_pipes, get_connector
+    from meerschaum import get_pipes
     from meerschaum.utils.debug import dprint
-    from meerschaum.utils.warnings import warn, info
-    from meerschaum.utils.misc import items_str
+    from meerschaum.utils.warnings import warn
     from meerschaum.config._patch import apply_patch_to_config
 
     if connector_keys is None:
@@ -164,10 +164,8 @@ def _register_plugins(
     from meerschaum.utils.debug import dprint
     from meerschaum.plugins import reload_plugins, get_plugins_names
     from meerschaum.connectors.parse import parse_repo_keys
-    from meerschaum.config import get_config
-    from meerschaum.utils.warnings import warn, error, info
+    from meerschaum.utils.warnings import warn, info
     from meerschaum.core import Plugin
-    from meerschaum import get_connector
     from meerschaum.utils.formatting import print_tuple
     from meerschaum.utils.prompt import prompt, yes_no
 
@@ -179,7 +177,7 @@ def _register_plugins(
     repo_connector = parse_repo_keys(repository)
     if repo_connector.__dict__.get('type', None) != 'api':
         return False, (
-            f"Can only upload plugins to the Meerschaum API." +
+            "Can only upload plugins to the Meerschaum API." +
             f"Connector '{repo_connector}' is of type " +
             f"'{repo_connector.get('type', type(repo_connector))}'."
         )
@@ -283,13 +281,11 @@ def _register_users(
     """
     from meerschaum.config import get_config
     from meerschaum.config.static import STATIC_CONFIG
-    from meerschaum import get_connector
     from meerschaum.connectors.parse import parse_instance_keys
-    from meerschaum.utils.debug import dprint
-    from meerschaum.utils.warnings import warn, error, info
+    from meerschaum.utils.warnings import warn, info
     from meerschaum.core import User
     from meerschaum.utils.formatting import print_tuple
-    from meerschaum.utils.prompt import prompt, get_password, get_email
+    from meerschaum.utils.prompt import get_password, get_email
     if mrsm_instance is None:
         mrsm_instance = get_config('meerschaum', 'instance')
     instance_connector = parse_instance_keys(mrsm_instance)
@@ -325,7 +321,7 @@ def _register_users(
                 minimum_length = STATIC_CONFIG['users']['min_password_length']
             )
             email = get_email(username, allow_omit=True)
-        except Exception as e:
+        except Exception:
             return False, (
                 "Aborted registering users " +
                 ', '.join(

meerschaum/actions/sql.py

@@ -127,7 +127,7 @@ def sql(
     
     from meerschaum.utils.packages import attempt_import
     from meerschaum.utils.formatting import print_tuple, pprint
-    _ = attempt_import('sqlalchemy.engine.result')
+    _ = attempt_import('sqlalchemy.engine.result', lazy=False)
     if 'sqlalchemy' in str(type(result)):
         if not nopretty:
             print_tuple((True, f"Successfully executed query:\n\n{query}"))

meerschaum/api/routes/_pipes.py

@@ -688,3 +688,21 @@ def get_pipe_columns_indices(
     Return a dictionary of column names and related indices.
     """
     return get_pipe(connector_keys, metric_key, location_key).get_columns_indices()
+
+
+@app.get(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/indices/names',
+    tags=['Pipes']
+)
+def get_pipe_index_names(
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> Dict[str, List[Dict[str, str]]]:
+    """
+    Return a dictionary of index keys and index names.
+    """
+    return get_pipe(connector_keys, metric_key, location_key).get_indices()

meerschaum/api/routes/_plugins.py

@@ -29,7 +29,7 @@ from meerschaum.core import Plugin
 starlette_responses = attempt_import('starlette.responses', warn=False)
 FileResponse = starlette_responses.FileResponse
 
-sqlalchemy = attempt_import('sqlalchemy')
+sqlalchemy = attempt_import('sqlalchemy', lazy=False)
 plugins_endpoint = endpoints['plugins']
 
 @app.post(plugins_endpoint + '/{name}', tags=['Plugins'])

meerschaum/api/routes/_users.py

@@ -7,36 +7,34 @@ Routes for managing users
 """
 
 from __future__ import annotations
+
 from meerschaum.utils.typing import (
-    Optional, Union, SuccessTuple, Any, Mapping, Sequence, Dict, List
+    Union, SuccessTuple, Any, Dict, List
 )
 
 from meerschaum.utils.packages import attempt_import
 from meerschaum.api import (
-    fastapi, app, endpoints, get_api_connector, pipes, get_pipe,
-    manager, debug, check_allow_chaining, DISALLOW_CHAINING_MESSAGE,
+    fastapi, app, endpoints, get_api_connector, manager,
+    debug, check_allow_chaining, DISALLOW_CHAINING_MESSAGE,
     no_auth, private,
 )
 from meerschaum.utils.misc import string_to_dict
 from meerschaum.config import get_config
-from meerschaum.api.tables import get_tables
-from starlette.responses import Response, JSONResponse
 from meerschaum.core import User
-import os, pathlib, datetime
 
-import meerschaum.core
-sqlalchemy = attempt_import('sqlalchemy')
+sqlalchemy = attempt_import('sqlalchemy', lazy=False)
 users_endpoint = endpoints['users']
 
 import fastapi
 from fastapi import HTTPException, Form
 
+
 @app.get(users_endpoint + "/me", tags=['Users'])
 def read_current_user(
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> Dict[str, Union[str, int]]:
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> Dict[str, Union[str, int]]:
     """
     Get information about the currently logged-in user.
     """
@@ -58,12 +56,13 @@ def read_current_user(
         ),
     }
 
+
 @app.get(users_endpoint, tags=['Users'])
 def get_users(
-        curr_user = (
-            fastapi.Depends(manager) if private else None
-        ),
-    ) -> List[str]:
+    curr_user = (
+        fastapi.Depends(manager) if private else None
+    ),
+) -> List[str]:
     """
     Get a list of the registered users.
     """
@@ -72,15 +71,15 @@ def get_users(
 
 @app.post(users_endpoint + "/register", tags=['Users'])
 def register_user(
-        username: str = Form(None),
-        password: str = Form(None),
-        attributes: str = Form(None),
-        type: str = Form(None),
-        email: str = Form(None),
-        curr_user = (
-            fastapi.Depends(manager) if private else None
-        ),
-    ) -> SuccessTuple:
+    username: str = Form(None),
+    password: str = Form(None),
+    attributes: str = Form(None),
+    type: str = Form(None),
+    email: str = Form(None),
+    curr_user = (
+        fastapi.Depends(manager) if private else None
+    ),
+) -> SuccessTuple:
     """
     Register a new user.
     """
@@ -90,7 +89,7 @@ def register_user(
     if attributes is not None:
         try:
             attributes = string_to_dict(attributes)
-        except Exception as e:
+        except Exception:
             return False, f"Invalid dictionary string received for attributes."
 
     allow_users = get_config('system', 'api', 'permissions', 'registration', 'users')
@@ -114,22 +113,22 @@ def register_user(
 
 @app.post(users_endpoint + "/edit", tags=['Users'])
 def edit_user(
-        username: str = Form(None),
-        password: str = Form(None),
-        type: str = Form(None),
-        email: str = Form(None),
-        attributes: str = Form(None),
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> SuccessTuple:
+    username: str = Form(None),
+    password: str = Form(None),
+    type: str = Form(None),
+    email: str = Form(None),
+    attributes: str = Form(None),
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> SuccessTuple:
     """
     Edit an existing user.
     """
     if attributes is not None:
         try:
             attributes = string_to_dict(attributes)
-        except Exception as e:
+        except Exception:
             return False, f"Invalid dictionary string received for attributes."
 
     user = User(username, password, email=email, attributes=attributes)
@@ -144,11 +143,11 @@ def edit_user(
 
 @app.get(users_endpoint + "/{username}/id", tags=['Users'])
 def get_user_id(
-        username : str,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> Union[int, None]:
+    username : str,
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> Union[int, None]:
     """
     Get a user's ID.
     """
@@ -157,23 +156,24 @@ def get_user_id(
 
 @app.get(users_endpoint + "/{username}/attributes", tags=['Users'])
 def get_user_attributes(
-        username : str,
-        curr_user = (
-            fastapi.Depends(manager) if private else None
-        ),
-    ) -> Union[Dict[str, Any], None]:
+    username : str,
+    curr_user = (
+        fastapi.Depends(manager) if private else None
+    ),
+) -> Union[Dict[str, Any], None]:
     """
     Get a user's attributes.
     """
     return get_api_connector().get_user_attributes(User(username), debug=debug)
 
+
 @app.delete(users_endpoint + "/{username}", tags=['Users'])
 def delete_user(
-        username: str,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> SuccessTuple:
+    username: str,
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> SuccessTuple:
     """
     Delete a user.
     """
@@ -193,11 +193,11 @@ def delete_user(
 
 @app.get(users_endpoint + '/{username}/password_hash', tags=['Users'])
 def get_user_password_hash(
-        username: str,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> str:
+    username: str,
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> str:
     """
     If configured to allow chaining, return a user's password_hash.
     """
@@ -205,13 +205,14 @@ def get_user_password_hash(
         raise HTTPException(status_code=403, detail=DISALLOW_CHAINING_MESSAGE)
     return get_api_connector().get_user_password_hash(User(username), debug=debug)
 
+
 @app.get(users_endpoint + '/{username}/type', tags=['Users'])
 def get_user_type(
-        username : str,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> str:
+    username : str,
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> str:
     """
     If configured to allow chaining, return a user's type.
     """

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.7.6"
+__version__ = "2.7.7"

meerschaum/connectors/api/_pipes.py

@@ -757,3 +757,23 @@ def get_pipe_columns_indices(
         warn(response.text)
         return None
     return j
+
+
+def get_pipe_index_names(self, pipe: mrsm.Pipe, debug: bool = False) -> Dict[str, str]:
+    """
+    Return the templated index names.
+    """
+    r_url = pipe_r_url(pipe) + '/indices/names'
+    response = self.get(
+        r_url,
+        debug=debug
+    )
+    j = response.json()
+    if isinstance(j, dict) and 'detail' in j and len(j.keys()) == 1:
+        warn(j['detail'])
+        return None
+    if not isinstance(j, dict):
+        warn(response.text)
+        return None
+    return j
+

meerschaum/connectors/sql/_SQLConnector.py

@@ -70,6 +70,9 @@ class SQLConnector(Connector):
         create_pipe_table_from_df,
         get_pipe_columns_indices,
         get_temporary_target,
+        create_pipe_indices,
+        drop_pipe_indices,
+        get_pipe_index_names,
     )
     from ._plugins import (
         register_plugin,
@@ -222,7 +225,7 @@ class SQLConnector(Connector):
                 return None
 
             from meerschaum.utils.packages import attempt_import
-            sqlalchemy_orm = attempt_import('sqlalchemy.orm')
+            sqlalchemy_orm = attempt_import('sqlalchemy.orm', lazy=False)
             session_factory = sqlalchemy_orm.sessionmaker(self.engine)
             self._Session = sqlalchemy_orm.scoped_session(session_factory)
 
@@ -290,7 +293,7 @@ class SQLConnector(Connector):
         Return the metadata bound to this configured schema.
         """
         from meerschaum.utils.packages import attempt_import
-        sqlalchemy = attempt_import('sqlalchemy')
+        sqlalchemy = attempt_import('sqlalchemy', lazy=False)
         if '_metadata' not in self.__dict__:
             self._metadata = sqlalchemy.MetaData(schema=self.schema)
         return self._metadata
@@ -368,7 +371,7 @@ class SQLConnector(Connector):
             self.__dict__['schema'] = None
             return None
 
-        sqlalchemy = mrsm.attempt_import('sqlalchemy')
+        sqlalchemy = mrsm.attempt_import('sqlalchemy', lazy=False)
         _schema = sqlalchemy.inspect(self.engine).default_schema_name
         self.__dict__['schema'] = _schema
         return _schema

meerschaum/connectors/sql/_create_engine.py

@@ -186,7 +186,7 @@ def create_engine(
     """Create a sqlalchemy engine by building the engine string."""
     from meerschaum.utils.packages import attempt_import
     from meerschaum.utils.warnings import error, warn
-    sqlalchemy = attempt_import('sqlalchemy')
+    sqlalchemy = attempt_import('sqlalchemy', lazy=False)
     import urllib
     import copy
     ### Install and patch required drivers.