meerschaum 2.7.6__py3-none-any.whl → 2.7.8__py3-none-any.whl

meerschaum/connectors/sql/_SQLConnector.py

@@ -70,6 +70,9 @@ class SQLConnector(Connector):
         create_pipe_table_from_df,
         get_pipe_columns_indices,
         get_temporary_target,
+        create_pipe_indices,
+        drop_pipe_indices,
+        get_pipe_index_names,
     )
     from ._plugins import (
         register_plugin,
@@ -192,7 +195,8 @@ class SQLConnector(Connector):
         self._debug = debug
         ### Store the PID and thread at initialization
         ### so we can dispose of the Pool in child processes or threads.
-        import os, threading
+        import os
+        import threading
         self._pid = os.getpid()
         self._thread_ident = threading.current_thread().ident
         self._sessions = {}
@@ -222,7 +226,7 @@ class SQLConnector(Connector):
                 return None
 
             from meerschaum.utils.packages import attempt_import
-            sqlalchemy_orm = attempt_import('sqlalchemy.orm')
+            sqlalchemy_orm = attempt_import('sqlalchemy.orm', lazy=False)
             session_factory = sqlalchemy_orm.sessionmaker(self.engine)
             self._Session = sqlalchemy_orm.scoped_session(session_factory)
 
@@ -283,19 +287,17 @@ class SQLConnector(Connector):
             return ':memory:' not in self.URI
         return True
 
-
     @property
     def metadata(self):
         """
         Return the metadata bound to this configured schema.
         """
         from meerschaum.utils.packages import attempt_import
-        sqlalchemy = attempt_import('sqlalchemy')
+        sqlalchemy = attempt_import('sqlalchemy', lazy=False)
         if '_metadata' not in self.__dict__:
             self._metadata = sqlalchemy.MetaData(schema=self.schema)
         return self._metadata
 
-
     @property
     def instance_schema(self):
         """
@@ -303,14 +305,12 @@ class SQLConnector(Connector):
         """
         return self.schema
 
-
     @property
     def internal_schema(self):
         """
         Return the schema name for internal tables. 
         """
         from meerschaum.config.static import STATIC_CONFIG
-        from meerschaum.utils.packages import attempt_import
         from meerschaum.utils.sql import NO_SCHEMA_FLAVORS
         schema_name = self.__dict__.get('internal_schema', None) or (
             STATIC_CONFIG['sql']['internal_schema']
@@ -322,7 +322,6 @@ class SQLConnector(Connector):
             self._internal_schema = schema_name
         return self._internal_schema
 
-
     @property
     def db(self) -> Optional[databases.Database]:
         from meerschaum.utils.packages import attempt_import
@@ -339,7 +338,6 @@ class SQLConnector(Connector):
                 self._db = None
         return self._db
 
-
     @property
     def db_version(self) -> Union[str, None]:
         """
@@ -353,7 +351,6 @@ class SQLConnector(Connector):
         self._db_version = get_db_version(self)
         return self._db_version
 
-
     @property
     def schema(self) -> Union[str, None]:
         """
@@ -368,12 +365,11 @@ class SQLConnector(Connector):
             self.__dict__['schema'] = None
             return None
 
-        sqlalchemy = mrsm.attempt_import('sqlalchemy')
+        sqlalchemy = mrsm.attempt_import('sqlalchemy', lazy=False)
         _schema = sqlalchemy.inspect(self.engine).default_schema_name
         self.__dict__['schema'] = _schema
         return _schema
 
-
     def __getstate__(self):
         return self.__dict__
 

meerschaum/connectors/sql/_create_engine.py

@@ -186,7 +186,7 @@ def create_engine(
     """Create a sqlalchemy engine by building the engine string."""
     from meerschaum.utils.packages import attempt_import
     from meerschaum.utils.warnings import error, warn
-    sqlalchemy = attempt_import('sqlalchemy')
+    sqlalchemy = attempt_import('sqlalchemy', lazy=False)
     import urllib
     import copy
     ### Install and patch required drivers.

meerschaum/connectors/sql/_fetch.py

@@ -11,7 +11,7 @@ from __future__ import annotations
 from datetime import datetime, timedelta
 
 import meerschaum as mrsm
-from meerschaum.utils.typing import Optional, Union, Callable, Any, List, Dict
+from meerschaum.utils.typing import Optional, Union, Any, List, Dict
 
 
 def fetch(
@@ -20,7 +20,6 @@ def fetch(
     begin: Union[datetime, int, str, None] = '',
     end: Union[datetime, int, str, None] = None,
     check_existing: bool = True,
-    chunk_hook: Optional[Callable[['pd.DataFrame'], Any]] = None,
     chunksize: Optional[int] = -1,
     workers: Optional[int] = None,
     debug: bool = False,
@@ -53,15 +52,12 @@ def fetch(
     check_existing: bool, defult True
         If `False`, use a backtrack interval of 0 minutes.
 
-    chunk_hook: Callable[[pd.DataFrame], Any], default None
-        A function to pass to `SQLConnector.read()` that accepts a Pandas DataFrame.
-
     chunksize: Optional[int], default -1
-        How many rows to load into memory at once (when `chunk_hook` is provided).
+        How many rows to load into memory at once.
         Otherwise the entire result set is loaded into memory.
 
     workers: Optional[int], default None
-        How many threads to use when consuming the generator (when `chunk_hook is provided).
+        How many threads to use when consuming the generator.
         Defaults to the number of cores.
 
     debug: bool, default False
@@ -69,8 +65,7 @@ def fetch(
 
     Returns
     -------
-    A pandas DataFrame or `None`.
-    If `chunk_hook` is not None, return a list of the hook function's results.
+    A pandas DataFrame generator.
     """
     meta_def = self.get_pipe_metadef(
         pipe,
@@ -80,33 +75,13 @@ def fetch(
         debug=debug,
         **kw
     )
-    as_hook_results = chunk_hook is not None
     chunks = self.read(
         meta_def,
-        chunk_hook=chunk_hook,
-        as_hook_results=as_hook_results,
         chunksize=chunksize,
         workers=workers,
+        as_iterator=True,
         debug=debug,
     )
-    ### if sqlite, parse for datetimes
-    if not as_hook_results and self.flavor == 'sqlite':
-        from meerschaum.utils.dataframe import parse_df_datetimes
-        from meerschaum.utils.dtypes import are_dtypes_equal
-        ignore_cols = [
-            col
-            for col, dtype in pipe.dtypes.items()
-            if not are_dtypes_equal(str(dtype), 'datetime')
-        ]
-        return (
-            parse_df_datetimes(
-                chunk,
-                ignore_cols=ignore_cols,
-                strip_timezone=(pipe.tzinfo is None),
-                debug=debug,
-            )
-            for chunk in chunks
-        )
     return chunks
 
 
@@ -148,7 +123,6 @@ def get_pipe_metadef(
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.sql import sql_item_name, dateadd_str, build_where
     from meerschaum.utils.dtypes.sql import get_db_type_from_pd_type
-    from meerschaum.utils.misc import is_int
     from meerschaum.config import get_config
 
     dt_col = pipe.columns.get('datetime', None)
@@ -191,7 +165,7 @@ def get_pipe_metadef(
         else begin
     )
 
-    if begin and end and begin >= end:
+    if begin not in (None, '') and end is not None and begin >= end:
         begin = None
 
     if dt_name:
@@ -203,7 +177,7 @@ def get_pipe_metadef(
                 begin=begin,
                 db_type=db_dt_typ,
             )
-            if begin
+            if begin not in ('', None)
             else None
         )
         end_da = (
@@ -214,7 +188,7 @@ def get_pipe_metadef(
                 begin=end,
                 db_type=db_dt_typ,
             )
-            if end
+            if end is not None
             else None
         )
 
@@ -228,11 +202,7 @@ def get_pipe_metadef(
 
     has_where = 'where' in meta_def.lower()[meta_def.lower().rfind('definition'):]
     if dt_name and (begin_da or end_da):
-        definition_dt_name = (
-            dateadd_str(self.flavor, 'minute', 0, f"{definition_name}.{dt_name}", db_type=db_dt_typ)
-            if not is_int((begin_da or end_da))
-            else f"{definition_name}.{dt_name}"
-        )
+        definition_dt_name = f"{definition_name}.{dt_name}"
         meta_def += "\n" + ("AND" if has_where else "WHERE") + " "
         has_where = True
         if begin_da:

meerschaum/connectors/sql/_instance.py

@@ -25,7 +25,7 @@ def _log_temporary_tables_creation(
     """
     from meerschaum.utils.misc import items_str
     from meerschaum.connectors.sql.tables import get_tables
-    sqlalchemy = mrsm.attempt_import('sqlalchemy')
+    sqlalchemy = mrsm.attempt_import('sqlalchemy', lazy=False)
     temp_tables_table = get_tables(
         mrsm_instance=self,
         create=create,
@@ -86,7 +86,7 @@ def _drop_temporary_tables(self, debug: bool = False) -> SuccessTuple:
     """
     from meerschaum.utils.misc import items_str
     from meerschaum.connectors.sql.tables import get_tables
-    sqlalchemy = mrsm.attempt_import('sqlalchemy')
+    sqlalchemy = mrsm.attempt_import('sqlalchemy', lazy=False)
     temp_tables_table = get_tables(
         mrsm_instance=self,
         create=False,
@@ -150,7 +150,7 @@ def _drop_old_temporary_tables(
     """
     from meerschaum.config import get_config
     from meerschaum.connectors.sql.tables import get_tables
-    sqlalchemy = mrsm.attempt_import('sqlalchemy')
+    sqlalchemy = mrsm.attempt_import('sqlalchemy', lazy=False)
     temp_tables_table = get_tables(mrsm_instance=self, create=False, debug=debug)['temp_tables']
     last_check = getattr(self, '_stale_temporary_tables_check_timestamp', 0)
     now_ts = time.perf_counter()