meerschaum 2.7.5__py3-none-any.whl → 2.7.7__py3-none-any.whl

meerschaum/api/routes/_plugins.py

@@ -29,7 +29,7 @@ from meerschaum.core import Plugin
 starlette_responses = attempt_import('starlette.responses', warn=False)
 FileResponse = starlette_responses.FileResponse
 
-sqlalchemy = attempt_import('sqlalchemy')
+sqlalchemy = attempt_import('sqlalchemy', lazy=False)
 plugins_endpoint = endpoints['plugins']
 
 @app.post(plugins_endpoint + '/{name}', tags=['Plugins'])

meerschaum/api/routes/_users.py

@@ -7,36 +7,34 @@ Routes for managing users
 """
 
 from __future__ import annotations
+
 from meerschaum.utils.typing import (
-    Optional, Union, SuccessTuple, Any, Mapping, Sequence, Dict, List
+    Union, SuccessTuple, Any, Dict, List
 )
 
 from meerschaum.utils.packages import attempt_import
 from meerschaum.api import (
-    fastapi, app, endpoints, get_api_connector, pipes, get_pipe,
-    manager, debug, check_allow_chaining, DISALLOW_CHAINING_MESSAGE,
+    fastapi, app, endpoints, get_api_connector, manager,
+    debug, check_allow_chaining, DISALLOW_CHAINING_MESSAGE,
     no_auth, private,
 )
 from meerschaum.utils.misc import string_to_dict
 from meerschaum.config import get_config
-from meerschaum.api.tables import get_tables
-from starlette.responses import Response, JSONResponse
 from meerschaum.core import User
-import os, pathlib, datetime
 
-import meerschaum.core
-sqlalchemy = attempt_import('sqlalchemy')
+sqlalchemy = attempt_import('sqlalchemy', lazy=False)
 users_endpoint = endpoints['users']
 
 import fastapi
 from fastapi import HTTPException, Form
 
+
 @app.get(users_endpoint + "/me", tags=['Users'])
 def read_current_user(
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> Dict[str, Union[str, int]]:
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> Dict[str, Union[str, int]]:
     """
     Get information about the currently logged-in user.
     """
@@ -58,12 +56,13 @@ def read_current_user(
         ),
     }
 
+
 @app.get(users_endpoint, tags=['Users'])
 def get_users(
-        curr_user = (
-            fastapi.Depends(manager) if private else None
-        ),
-    ) -> List[str]:
+    curr_user = (
+        fastapi.Depends(manager) if private else None
+    ),
+) -> List[str]:
     """
     Get a list of the registered users.
     """
@@ -72,15 +71,15 @@ def get_users(
 
 @app.post(users_endpoint + "/register", tags=['Users'])
 def register_user(
-        username: str = Form(None),
-        password: str = Form(None),
-        attributes: str = Form(None),
-        type: str = Form(None),
-        email: str = Form(None),
-        curr_user = (
-            fastapi.Depends(manager) if private else None
-        ),
-    ) -> SuccessTuple:
+    username: str = Form(None),
+    password: str = Form(None),
+    attributes: str = Form(None),
+    type: str = Form(None),
+    email: str = Form(None),
+    curr_user = (
+        fastapi.Depends(manager) if private else None
+    ),
+) -> SuccessTuple:
     """
     Register a new user.
     """
@@ -90,7 +89,7 @@ def register_user(
     if attributes is not None:
         try:
             attributes = string_to_dict(attributes)
-        except Exception as e:
+        except Exception:
             return False, f"Invalid dictionary string received for attributes."
 
     allow_users = get_config('system', 'api', 'permissions', 'registration', 'users')
@@ -114,22 +113,22 @@ def register_user(
 
 @app.post(users_endpoint + "/edit", tags=['Users'])
 def edit_user(
-        username: str = Form(None),
-        password: str = Form(None),
-        type: str = Form(None),
-        email: str = Form(None),
-        attributes: str = Form(None),
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> SuccessTuple:
+    username: str = Form(None),
+    password: str = Form(None),
+    type: str = Form(None),
+    email: str = Form(None),
+    attributes: str = Form(None),
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> SuccessTuple:
     """
     Edit an existing user.
     """
     if attributes is not None:
         try:
             attributes = string_to_dict(attributes)
-        except Exception as e:
+        except Exception:
             return False, f"Invalid dictionary string received for attributes."
 
     user = User(username, password, email=email, attributes=attributes)
@@ -144,11 +143,11 @@ def edit_user(
 
 @app.get(users_endpoint + "/{username}/id", tags=['Users'])
 def get_user_id(
-        username : str,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> Union[int, None]:
+    username : str,
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> Union[int, None]:
     """
     Get a user's ID.
     """
@@ -157,23 +156,24 @@ def get_user_id(
 
 @app.get(users_endpoint + "/{username}/attributes", tags=['Users'])
 def get_user_attributes(
-        username : str,
-        curr_user = (
-            fastapi.Depends(manager) if private else None
-        ),
-    ) -> Union[Dict[str, Any], None]:
+    username : str,
+    curr_user = (
+        fastapi.Depends(manager) if private else None
+    ),
+) -> Union[Dict[str, Any], None]:
     """
     Get a user's attributes.
     """
     return get_api_connector().get_user_attributes(User(username), debug=debug)
 
+
 @app.delete(users_endpoint + "/{username}", tags=['Users'])
 def delete_user(
-        username: str,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> SuccessTuple:
+    username: str,
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> SuccessTuple:
     """
     Delete a user.
     """
@@ -193,11 +193,11 @@ def delete_user(
 
 @app.get(users_endpoint + '/{username}/password_hash', tags=['Users'])
 def get_user_password_hash(
-        username: str,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> str:
+    username: str,
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> str:
     """
     If configured to allow chaining, return a user's password_hash.
     """
@@ -205,13 +205,14 @@ def get_user_password_hash(
         raise HTTPException(status_code=403, detail=DISALLOW_CHAINING_MESSAGE)
     return get_api_connector().get_user_password_hash(User(username), debug=debug)
 
+
 @app.get(users_endpoint + '/{username}/type', tags=['Users'])
 def get_user_type(
-        username : str,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> str:
+    username : str,
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> str:
     """
     If configured to allow chaining, return a user's type.
     """

meerschaum/config/_default.py

@@ -76,6 +76,11 @@ default_system_config = {
             },
             'instance': {
                 'stale_temporary_tables_minutes': 1440,
+                'temporary_target': {
+                    'prefix': '_',
+                    'transaction_id_length': 4,
+                    'separator': '_',
+                },
             },
             'chunksize': 100_000,
             'poolclass': 'sqlalchemy.pool.QueuePool',

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.7.5"
+__version__ = "2.7.7"

meerschaum/connectors/api/_misc.py

@@ -20,12 +20,13 @@ def get_mrsm_version(self, **kw) -> Optional[str]:
             use_token=False,
             **kw
         ).json()
-    except Exception as e:
+    except Exception:
         return None
     if isinstance(j, dict) and 'detail' in j:
         return None
     return j
 
+
 def get_chaining_status(self, **kw) -> Optional[bool]:
     """
     Fetch the chaining status of the API instance.
@@ -39,7 +40,7 @@ def get_chaining_status(self, **kw) -> Optional[bool]:
         )
         if not response:
             return None
-    except Exception as e:
+    except Exception:
         return None
 
     return response.json()

meerschaum/connectors/api/_pipes.py

@@ -9,8 +9,7 @@ Register or fetch Pipes from the API
 from __future__ import annotations
 import time
 import json
-from io import StringIO
-from datetime import datetime
+from datetime import datetime, timedelta
 
 import meerschaum as mrsm
 from meerschaum.utils.debug import dprint
@@ -178,11 +177,11 @@ def sync_pipe(
     """Sync a DataFrame into a Pipe."""
     from decimal import Decimal
     from meerschaum.utils.debug import dprint
-    from meerschaum.utils.misc import json_serialize_datetime, items_str
+    from meerschaum.utils.misc import json_serialize_datetime, items_str, interval_str
     from meerschaum.config import get_config
     from meerschaum.utils.packages import attempt_import
-    from meerschaum.utils.dataframe import get_numeric_cols, to_json, get_bytes_cols
-    begin = time.time()
+    from meerschaum.utils.dataframe import get_numeric_cols, to_json
+    begin = time.perf_counter()
     more_itertools = attempt_import('more_itertools')
     if df is None:
         msg = f"DataFrame is `None`. Cannot sync {pipe}."
@@ -304,9 +303,10 @@ def sync_pipe(
         num_success_chunks += 1
 
     success_tuple = True, (
-        f"It took {round(time.time() - begin, 2)} seconds to sync {rowcount} row"
+        f"It took {interval_str(timedelta(seconds=(time.perf_counter() - begin)))} "
+        + "to sync {rowcount:,} row"
         + ('s' if rowcount != 1 else '')
-        + f" across {num_success_chunks} chunk" + ('s' if num_success_chunks != 1 else '') +
+        + f" across {num_success_chunks:,} chunk" + ('s' if num_success_chunks != 1 else '') +
         f" to {pipe}."
     )
     return success_tuple
@@ -549,10 +549,9 @@ def create_metadata(
     if debug:
         dprint("Create metadata response: {response.text}")
     try:
-        metadata_response = json.loads(response.text)
+        _ = json.loads(response.text)
     except Exception as e:
         warn(f"Failed to create metadata on {self}:\n{e}")
-        metadata_response = False
     return False
 
 
@@ -758,3 +757,23 @@ def get_pipe_columns_indices(
         warn(response.text)
         return None
     return j
+
+
+def get_pipe_index_names(self, pipe: mrsm.Pipe, debug: bool = False) -> Dict[str, str]:
+    """
+    Return the templated index names.
+    """
+    r_url = pipe_r_url(pipe) + '/indices/names'
+    response = self.get(
+        r_url,
+        debug=debug
+    )
+    j = response.json()
+    if isinstance(j, dict) and 'detail' in j and len(j.keys()) == 1:
+        warn(j['detail'])
+        return None
+    if not isinstance(j, dict):
+        warn(response.text)
+        return None
+    return j
+

meerschaum/connectors/sql/_SQLConnector.py

@@ -69,6 +69,10 @@ class SQLConnector(Connector):
         get_pipe_schema,
         create_pipe_table_from_df,
         get_pipe_columns_indices,
+        get_temporary_target,
+        create_pipe_indices,
+        drop_pipe_indices,
+        get_pipe_index_names,
     )
     from ._plugins import (
         register_plugin,
@@ -221,7 +225,7 @@ class SQLConnector(Connector):
                 return None
 
             from meerschaum.utils.packages import attempt_import
-            sqlalchemy_orm = attempt_import('sqlalchemy.orm')
+            sqlalchemy_orm = attempt_import('sqlalchemy.orm', lazy=False)
             session_factory = sqlalchemy_orm.sessionmaker(self.engine)
             self._Session = sqlalchemy_orm.scoped_session(session_factory)
 
@@ -289,7 +293,7 @@ class SQLConnector(Connector):
         Return the metadata bound to this configured schema.
         """
         from meerschaum.utils.packages import attempt_import
-        sqlalchemy = attempt_import('sqlalchemy')
+        sqlalchemy = attempt_import('sqlalchemy', lazy=False)
         if '_metadata' not in self.__dict__:
             self._metadata = sqlalchemy.MetaData(schema=self.schema)
         return self._metadata
@@ -367,7 +371,7 @@ class SQLConnector(Connector):
             self.__dict__['schema'] = None
             return None
 
-        sqlalchemy = mrsm.attempt_import('sqlalchemy')
+        sqlalchemy = mrsm.attempt_import('sqlalchemy', lazy=False)
         _schema = sqlalchemy.inspect(self.engine).default_schema_name
         self.__dict__['schema'] = _schema
         return _schema

meerschaum/connectors/sql/_create_engine.py

@@ -186,7 +186,7 @@ def create_engine(
     """Create a sqlalchemy engine by building the engine string."""
     from meerschaum.utils.packages import attempt_import
     from meerschaum.utils.warnings import error, warn
-    sqlalchemy = attempt_import('sqlalchemy')
+    sqlalchemy = attempt_import('sqlalchemy', lazy=False)
     import urllib
     import copy
     ### Install and patch required drivers.

meerschaum/connectors/sql/_fetch.py

@@ -148,7 +148,6 @@ def get_pipe_metadef(
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.sql import sql_item_name, dateadd_str, build_where
     from meerschaum.utils.dtypes.sql import get_db_type_from_pd_type
-    from meerschaum.utils.misc import is_int
     from meerschaum.config import get_config
 
     dt_col = pipe.columns.get('datetime', None)
@@ -191,7 +190,7 @@ def get_pipe_metadef(
         else begin
     )
 
-    if begin and end and begin >= end:
+    if begin not in (None, '') and end is not None and begin >= end:
         begin = None
 
     if dt_name:
@@ -203,7 +202,7 @@ def get_pipe_metadef(
                 begin=begin,
                 db_type=db_dt_typ,
             )
-            if begin
+            if begin not in ('', None)
             else None
         )
         end_da = (
@@ -214,7 +213,7 @@ def get_pipe_metadef(
                 begin=end,
                 db_type=db_dt_typ,
             )
-            if end
+            if end is not None
             else None
         )
 
@@ -228,11 +227,7 @@ def get_pipe_metadef(
 
     has_where = 'where' in meta_def.lower()[meta_def.lower().rfind('definition'):]
     if dt_name and (begin_da or end_da):
-        definition_dt_name = (
-            dateadd_str(self.flavor, 'minute', 0, f"{definition_name}.{dt_name}", db_type=db_dt_typ)
-            if not is_int((begin_da or end_da))
-            else f"{definition_name}.{dt_name}"
-        )
+        definition_dt_name = f"{definition_name}.{dt_name}"
         meta_def += "\n" + ("AND" if has_where else "WHERE") + " "
         has_where = True
         if begin_da:

meerschaum/connectors/sql/_instance.py

@@ -25,7 +25,7 @@ def _log_temporary_tables_creation(
     """
     from meerschaum.utils.misc import items_str
     from meerschaum.connectors.sql.tables import get_tables
-    sqlalchemy = mrsm.attempt_import('sqlalchemy')
+    sqlalchemy = mrsm.attempt_import('sqlalchemy', lazy=False)
     temp_tables_table = get_tables(
         mrsm_instance=self,
         create=create,
@@ -86,7 +86,7 @@ def _drop_temporary_tables(self, debug: bool = False) -> SuccessTuple:
     """
     from meerschaum.utils.misc import items_str
     from meerschaum.connectors.sql.tables import get_tables
-    sqlalchemy = mrsm.attempt_import('sqlalchemy')
+    sqlalchemy = mrsm.attempt_import('sqlalchemy', lazy=False)
     temp_tables_table = get_tables(
         mrsm_instance=self,
         create=False,
@@ -150,7 +150,7 @@ def _drop_old_temporary_tables(
     """
     from meerschaum.config import get_config
     from meerschaum.connectors.sql.tables import get_tables
-    sqlalchemy = mrsm.attempt_import('sqlalchemy')
+    sqlalchemy = mrsm.attempt_import('sqlalchemy', lazy=False)
     temp_tables_table = get_tables(mrsm_instance=self, create=False, debug=debug)['temp_tables']
     last_check = getattr(self, '_stale_temporary_tables_check_timestamp', 0)
     now_ts = time.perf_counter()