meerschaum 2.9.4__py3-none-any.whl → 3.0.0__py3-none-any.whl

meerschaum/connectors/sql/_plugins.py

@@ -13,6 +13,35 @@ import json
 import meerschaum as mrsm
 from meerschaum.utils.typing import Optional, Any, List, SuccessTuple, Dict
 
+
+def get_plugins_pipe(self) -> mrsm.Pipe:
+    """
+    Return the internal metadata plugins pipe.
+    """
+    users_pipe = self.get_users_pipe()
+    user_id_dtype = users_pipe.dtypes.get('user_id', 'int')
+    return mrsm.Pipe(
+        'mrsm', 'plugins',
+        instance=self,
+        temporary=True,
+        static=True,
+        null_indices=False,
+        columns={
+            'primary': 'plugin_id',
+            'user_id': 'user_id',    
+        },
+        dtypes={
+            'plugin_name': 'string',
+            'user_id': user_id_dtype,
+            'attributes': 'json',
+            'version': 'string',
+        },
+        indices={
+            'unique': 'plugin_name',
+        },
+    )
+
+
 def register_plugin(
     self,
     plugin: 'mrsm.core.Plugin',

meerschaum/connectors/sql/_sql.py

@@ -17,7 +17,14 @@ from meerschaum.utils.debug import dprint
 from meerschaum.utils.warnings import warn
 
 ### database flavors that can use bulk insert
-_bulk_flavors = {'postgresql', 'postgis', 'timescaledb', 'citus', 'mssql'}
+_bulk_flavors = {
+    'postgresql',
+    'postgis',
+    'timescaledb',
+    'timescaledb-ha',
+    'citus',
+    'mssql',
+}
 ### flavors that do not support chunks
 _disallow_chunks_flavors = ['duckdb']
 _max_chunks_flavors = {'sqlite': 1000}
@@ -124,23 +131,28 @@ def read(
     """
     if chunks is not None and chunks <= 0:
         return []
+
     from meerschaum.utils.sql import sql_item_name, truncate_item_name
     from meerschaum.utils.dtypes import are_dtypes_equal, coerce_timezone
     from meerschaum.utils.dtypes.sql import TIMEZONE_NAIVE_FLAVORS
     from meerschaum.utils.packages import attempt_import, import_pandas
     from meerschaum.utils.pool import get_pool
     from meerschaum.utils.dataframe import chunksize_to_npartitions, get_numeric_cols
+    from meerschaum.utils.misc import filter_arguments
     import warnings
     import traceback
     from decimal import Decimal
+
     pd = import_pandas()
     dd = None
+
     is_dask = 'dask' in pd.__name__
     pandas = attempt_import('pandas')
     is_dask = dd is not None
     npartitions = chunksize_to_npartitions(chunksize)
     if is_dask:
         chunksize = None
+
     schema = schema or self.schema
     utc_dt_cols = [
         col
@@ -151,7 +163,7 @@ def read(
     if dtype and utc_dt_cols and self.flavor in TIMEZONE_NAIVE_FLAVORS:
         dtype = dtype.copy()
         for col in utc_dt_cols:
-            dtype[col] = 'datetime64[ns]'
+            dtype[col] = 'datetime64[us]'
 
     pool = get_pool(workers=workers)
     sqlalchemy = attempt_import("sqlalchemy", lazy=False)
@@ -215,26 +227,33 @@ def read(
         else format_sql_query_for_dask(str_query)
     )
 
+    def _get_chunk_args_kwargs(_chunk):
+        return filter_arguments(
+            chunk_hook,
+            _chunk,
+            workers=workers,
+            chunksize=chunksize,
+            debug=debug,
+            **kw
+        )
+
     chunk_list = []
     chunk_hook_results = []
     def _process_chunk(_chunk, _retry_on_failure: bool = True):
         if self.flavor in TIMEZONE_NAIVE_FLAVORS:
             for col in utc_dt_cols:
-                _chunk[col] = coerce_timezone(_chunk[col], strip_timezone=False)
+                _chunk[col] = coerce_timezone(_chunk[col], strip_utc=False)
         if not as_hook_results:
             chunk_list.append(_chunk)
+
         if chunk_hook is None:
             return None
 
+        chunk_args, chunk_kwargs = _get_chunk_args_kwargs(_chunk)
+
         result = None
         try:
-            result = chunk_hook(
-                _chunk,
-                workers=workers,
-                chunksize=chunksize,
-                debug=debug,
-                **kw
-            )
+            result = chunk_hook(*chunk_args, **chunk_kwargs)
         except Exception:
             result = False, traceback.format_exc()
             from meerschaum.utils.formatting import get_console
@@ -285,8 +304,16 @@ def read(
                         self.engine,
                         **read_sql_query_kwargs
                     )
+
                     to_return = (
-                        chunk_generator
+                        (
+                            chunk_generator
+                            if not (as_hook_results or chunksize is None)
+                            else (
+                                _process_chunk(_chunk)
+                                for _chunk in chunk_generator
+                            )
+                        )
                         if as_iterator or chunksize is None
                         else (
                             list(pool.imap(_process_chunk, chunk_generator))
@@ -332,9 +359,8 @@ def read(
         try:
             for chunk in chunk_generator:
                 if chunk_hook is not None:
-                    chunk_hook_results.append(
-                        chunk_hook(chunk, chunksize=chunksize, debug=debug, **kw)
-                    )
+                    chunk_args, chunk_kwargs = _get_chunk_args_kwargs(chunk)
+                    chunk_hook_results.append(chunk_hook(*chunk_args, **chunk_kwargs))
                 chunk_list.append(chunk)
                 read_chunks += 1
                 if chunks is not None and read_chunks >= chunks:
@@ -349,9 +375,8 @@ def read(
     try:
         for chunk in chunk_generator:
             if chunk_hook is not None:
-                chunk_hook_results.append(
-                    chunk_hook(chunk, chunksize=chunksize, debug=debug, **kw)
-                )
+                chunk_args, chunk_kwargs = _get_chunk_args_kwargs(chunk)
+                chunk_hook_results.append(chunk_hook(*chunk_args, **chunk_kwargs))
             chunk_list.append(chunk)
             read_chunks += 1
             if chunks is not None and read_chunks >= chunks:
@@ -382,9 +407,8 @@ def read(
     ### call the hook on any missed chunks.
     if chunk_hook is not None and len(chunk_list) > len(chunk_hook_results):
         for c in chunk_list[len(chunk_hook_results):]:
-            chunk_hook_results.append(
-                chunk_hook(c, chunksize=chunksize, debug=debug, **kw)
-            )
+            chunk_args, chunk_kwargs = _get_chunk_args_kwargs(c)
+            chunk_hook_results.append(chunk_hook(*chunk_args, **chunk_kwargs))
 
     ### chunksize is not None so must iterate
     if debug:
@@ -777,6 +801,7 @@ def to_sql(
     from meerschaum.utils.warnings import error, warn
     import warnings
     import functools
+    import traceback
 
     if name is None:
         error(f"Name must not be `None` to insert data into {self}.")
@@ -928,7 +953,7 @@ def to_sql(
             df[col] = df[col].apply(
                 functools.partial(
                     serialize_geometry,
-                    as_wkt=(self.flavor == 'mssql')
+                    geometry_format=('wkt' if self.flavor == 'mssql' else 'wkb_hex'),
                 )
             )
 
@@ -1050,7 +1075,7 @@ def to_sql(
     except Exception as e:
         if not silent:
             warn(str(e))
-        success, msg = False, str(e)
+        success, msg = False, traceback.format_exc()
 
     end = time.perf_counter()
     if success:

meerschaum/connectors/sql/_users.py

@@ -12,6 +12,39 @@ import meerschaum as mrsm
 from meerschaum.utils.typing import SuccessTuple, Optional, Any, Dict, List, Union
 
 
+def get_users_pipe(self) -> mrsm.Pipe:
+    """
+    Return the internal metadata pipe for users management.
+    """
+    if '_users_pipe' in self.__dict__:
+        return self._users_pipe
+
+    cache_connector = self.__dict__.get('_cache_connector', None)
+    self._users_pipe = mrsm.Pipe(
+        'mrsm', 'users',
+        temporary=True,
+        cache=True,
+        cache_connector_keys=cache_connector,
+        static=True,
+        null_indices=False,
+        enforce=False,
+        autoincrement=True,
+        columns={
+            'primary': 'user_id',
+        },
+        dtypes={
+            'user_id': 'int',
+            'username': 'string',
+            'attributes': 'json',
+            'user_type': 'string',
+        },
+        indices={
+            'unique': 'username',
+        },
+    )
+    return self._users_pipe
+
+
 def register_user(
     self,
     user: mrsm.core.User,
@@ -64,7 +97,7 @@ def register_user(
 
 def valid_username(username: str) -> SuccessTuple:
     """Verify that a given username is valid."""
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     fail_reasons = []
 
     min_length = STATIC_CONFIG['users']['min_username_length']
@@ -104,6 +137,7 @@ def edit_user(
 ) -> SuccessTuple:
     """Update an existing user's metadata."""
     from meerschaum.utils.packages import attempt_import
+    from meerschaum.utils.sql import json_flavors
     sqlalchemy = attempt_import('sqlalchemy', lazy=False)
     from meerschaum.connectors.sql.tables import get_tables
     users_tbl = get_tables(mrsm_instance=self, debug=debug)['users']
@@ -131,7 +165,7 @@ def edit_user(
         bind_variables['email'] = user.email
     if user.attributes is not None and user.attributes != {}:
         bind_variables['attributes'] = (
-            json.dumps(user.attributes) if self.flavor in ('duckdb',)
+            json.dumps(user.attributes) if self.flavor not in json_flavors
             else user.attributes
         )
     if user.type != '':