meerschaum 2.9.5__py3-none-any.whl → 3.0.0rc2__py3-none-any.whl

meerschaum/connectors/sql/_sql.py

@@ -17,7 +17,14 @@ from meerschaum.utils.debug import dprint
 from meerschaum.utils.warnings import warn
 
 ### database flavors that can use bulk insert
-_bulk_flavors = {'postgresql', 'postgis', 'timescaledb', 'citus', 'mssql'}
+_bulk_flavors = {
+    'postgresql',
+    'postgis',
+    'timescaledb',
+    'timescaledb-ha',
+    'citus',
+    'mssql',
+}
 ### flavors that do not support chunks
 _disallow_chunks_flavors = ['duckdb']
 _max_chunks_flavors = {'sqlite': 1000}
@@ -124,23 +131,28 @@ def read(
     """
     if chunks is not None and chunks <= 0:
         return []
+
     from meerschaum.utils.sql import sql_item_name, truncate_item_name
     from meerschaum.utils.dtypes import are_dtypes_equal, coerce_timezone
     from meerschaum.utils.dtypes.sql import TIMEZONE_NAIVE_FLAVORS
     from meerschaum.utils.packages import attempt_import, import_pandas
     from meerschaum.utils.pool import get_pool
     from meerschaum.utils.dataframe import chunksize_to_npartitions, get_numeric_cols
+    from meerschaum.utils.misc import filter_arguments
     import warnings
     import traceback
     from decimal import Decimal
+
     pd = import_pandas()
     dd = None
+
     is_dask = 'dask' in pd.__name__
     pandas = attempt_import('pandas')
     is_dask = dd is not None
     npartitions = chunksize_to_npartitions(chunksize)
     if is_dask:
         chunksize = None
+
     schema = schema or self.schema
     utc_dt_cols = [
         col
@@ -151,7 +163,7 @@ def read(
     if dtype and utc_dt_cols and self.flavor in TIMEZONE_NAIVE_FLAVORS:
         dtype = dtype.copy()
         for col in utc_dt_cols:
-            dtype[col] = 'datetime64[ns]'
+            dtype[col] = 'datetime64[us]'
 
     pool = get_pool(workers=workers)
     sqlalchemy = attempt_import("sqlalchemy", lazy=False)
@@ -215,26 +227,33 @@ def read(
         else format_sql_query_for_dask(str_query)
     )
 
+    def _get_chunk_args_kwargs(_chunk):
+        return filter_arguments(
+            chunk_hook,
+            _chunk,
+            workers=workers,
+            chunksize=chunksize,
+            debug=debug,
+            **kw
+        )
+
     chunk_list = []
     chunk_hook_results = []
     def _process_chunk(_chunk, _retry_on_failure: bool = True):
         if self.flavor in TIMEZONE_NAIVE_FLAVORS:
             for col in utc_dt_cols:
-                _chunk[col] = coerce_timezone(_chunk[col], strip_timezone=False)
+                _chunk[col] = coerce_timezone(_chunk[col], strip_utc=False)
         if not as_hook_results:
             chunk_list.append(_chunk)
+
         if chunk_hook is None:
             return None
 
+        chunk_args, chunk_kwargs = _get_chunk_args_kwargs(_chunk)
+
         result = None
         try:
-            result = chunk_hook(
-                _chunk,
-                workers=workers,
-                chunksize=chunksize,
-                debug=debug,
-                **kw
-            )
+            result = chunk_hook(*chunk_args, **chunk_kwargs)
         except Exception:
             result = False, traceback.format_exc()
             from meerschaum.utils.formatting import get_console
@@ -285,8 +304,16 @@ def read(
                         self.engine,
                         **read_sql_query_kwargs
                     )
+
                     to_return = (
-                        chunk_generator
+                        (
+                            chunk_generator
+                            if not (as_hook_results or chunksize is None)
+                            else (
+                                _process_chunk(_chunk)
+                                for _chunk in chunk_generator
+                            )
+                        )
                         if as_iterator or chunksize is None
                         else (
                             list(pool.imap(_process_chunk, chunk_generator))
@@ -332,9 +359,8 @@ def read(
         try:
             for chunk in chunk_generator:
                 if chunk_hook is not None:
-                    chunk_hook_results.append(
-                        chunk_hook(chunk, chunksize=chunksize, debug=debug, **kw)
-                    )
+                    chunk_args, chunk_kwargs = _get_chunk_args_kwargs(chunk)
+                    chunk_hook_results.append(chunk_hook(*chunk_args, **chunk_kwargs))
                 chunk_list.append(chunk)
                 read_chunks += 1
                 if chunks is not None and read_chunks >= chunks:
@@ -349,9 +375,8 @@ def read(
     try:
         for chunk in chunk_generator:
             if chunk_hook is not None:
-                chunk_hook_results.append(
-                    chunk_hook(chunk, chunksize=chunksize, debug=debug, **kw)
-                )
+                chunk_args, chunk_kwargs = _get_chunk_args_kwargs(chunk)
+                chunk_hook_results.append(chunk_hook(*chunk_args, **chunk_kwargs))
             chunk_list.append(chunk)
             read_chunks += 1
             if chunks is not None and read_chunks >= chunks:
@@ -382,9 +407,8 @@ def read(
     ### call the hook on any missed chunks.
     if chunk_hook is not None and len(chunk_list) > len(chunk_hook_results):
         for c in chunk_list[len(chunk_hook_results):]:
-            chunk_hook_results.append(
-                chunk_hook(c, chunksize=chunksize, debug=debug, **kw)
-            )
+            chunk_args, chunk_kwargs = _get_chunk_args_kwargs(c)
+            chunk_hook_results.append(chunk_hook(*chunk_args, **chunk_kwargs))
 
     ### chunksize is not None so must iterate
     if debug:
@@ -777,6 +801,7 @@ def to_sql(
     from meerschaum.utils.warnings import error, warn
     import warnings
     import functools
+    import traceback
 
     if name is None:
         error(f"Name must not be `None` to insert data into {self}.")
@@ -1050,7 +1075,7 @@ def to_sql(
     except Exception as e:
         if not silent:
             warn(str(e))
-        success, msg = False, str(e)
+        success, msg = False, traceback.format_exc()
 
     end = time.perf_counter()
     if success:

meerschaum/connectors/sql/_users.py

@@ -12,6 +12,32 @@ import meerschaum as mrsm
 from meerschaum.utils.typing import SuccessTuple, Optional, Any, Dict, List, Union
 
 
+def get_users_pipe(self) -> mrsm.Pipe:
+    """
+    Return the internal metadata pipe for users management.
+    """
+    return mrsm.Pipe(
+        'mrsm', 'users',
+        temporary=True,
+        static=True,
+        null_indices=False,
+        enforce=False,
+        autoincrement=True,
+        columns={
+            'primary': 'user_id',
+        },
+        dtypes={
+            'user_id': 'int',
+            'username': 'string',
+            'attributes': 'json',
+            'user_type': 'string',
+        },
+        indices={
+            'unique': 'username',
+        },
+    )
+
+
 def register_user(
     self,
     user: mrsm.core.User,
@@ -64,7 +90,7 @@ def register_user(
 
 def valid_username(username: str) -> SuccessTuple:
     """Verify that a given username is valid."""
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     fail_reasons = []
 
     min_length = STATIC_CONFIG['users']['min_username_length']
@@ -104,6 +130,7 @@ def edit_user(
 ) -> SuccessTuple:
     """Update an existing user's metadata."""
     from meerschaum.utils.packages import attempt_import
+    from meerschaum.utils.sql import json_flavors
     sqlalchemy = attempt_import('sqlalchemy', lazy=False)
     from meerschaum.connectors.sql.tables import get_tables
     users_tbl = get_tables(mrsm_instance=self, debug=debug)['users']
@@ -131,7 +158,7 @@ def edit_user(
         bind_variables['email'] = user.email
     if user.attributes is not None and user.attributes != {}:
         bind_variables['attributes'] = (
-            json.dumps(user.attributes) if self.flavor in ('duckdb',)
+            json.dumps(user.attributes) if self.flavor not in json_flavors
             else user.attributes
         )
     if user.type != '':

meerschaum/connectors/sql/tables/__init__.py

@@ -231,7 +231,7 @@ def create_schemas(
     """
     Create the internal Meerschaum schema on the database.
     """
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     from meerschaum.utils.packages import attempt_import
     from meerschaum.utils.sql import sql_item_name, NO_SCHEMA_FLAVORS, SKIP_IF_EXISTS_FLAVORS
     if conn.flavor in NO_SCHEMA_FLAVORS:

meerschaum/connectors/valkey/_ValkeyConnector.py

@@ -10,19 +10,18 @@ import json
 from datetime import datetime, timezone
 
 import meerschaum as mrsm
-from meerschaum.connectors import Connector, make_connector
+from meerschaum.connectors import InstanceConnector, make_connector
 from meerschaum.utils.typing import List, Dict, Any, Optional, Union
 from meerschaum.utils.warnings import dprint
 
 
 @make_connector
-class ValkeyConnector(Connector):
+class ValkeyConnector(InstanceConnector):
     """
     Manage a Valkey instance.
 
     Build a `ValkeyConnector` from connection attributes or a URI string.
     """
-    IS_INSTANCE: bool = True
     REQUIRED_ATTRIBUTES: List[str] = ['host']
     OPTIONAL_ATTRIBUTES: List[str] = [
         'port', 'username', 'password', 'db', 'socket_timeout',
@@ -80,7 +79,6 @@ class ValkeyConnector(Connector):
         get_plugin_user_id,
         get_plugin_username,
         get_plugin_attributes,
-        get_plugins,
         delete_plugin,
     )
 

meerschaum/connectors/valkey/_pipes.py

@@ -13,7 +13,7 @@ from meerschaum.utils.typing import SuccessTuple, Any, Union, Optional, Dict, Li
 from meerschaum.utils.misc import string_to_dict
 from meerschaum.utils.dtypes import json_serialize_value
 from meerschaum.utils.warnings import warn, dprint
-from meerschaum.config.static import STATIC_CONFIG
+from meerschaum._internal.static import STATIC_CONFIG
 
 PIPES_TABLE: str = 'mrsm_pipes'
 PIPES_COUNTER: str = 'mrsm_pipes:counter'
@@ -276,7 +276,7 @@ def edit_pipe(
         return False, f"{pipe} is not registered."
 
     parameters_key = get_pipe_parameters_key(pipe)
-    parameters_str = json.dumps(pipe.parameters, separators=(',', ':'))
+    parameters_str = json.dumps(pipe.get_parameters(apply_symlinks=False), separators=(',', ':'))
     self.set(parameters_key, parameters_str)
     return True, "Success"
 
@@ -321,14 +321,40 @@ def drop_pipe(
     -------
     A `SuccessTuple` indicating success.
     """
-    for chunk_begin, chunk_end in pipe.get_chunk_bounds(debug=debug):
-        clear_chunk_success, clear_chunk_msg = pipe.clear(
-            begin=chunk_begin,
-            end=chunk_end,
-            debug=debug,
+    if not pipe.exists(debug=debug):
+        return True, f"{pipe} does not exist, so it was not dropped."
+
+    table_name = self.quote_table(pipe.target)
+    dt_col = pipe.columns.get('datetime', None)
+
+    try:
+        members = (
+            self.client.zrange(table_name, 0, -1)
+            if dt_col
+            else self.client.smembers(table_name)
         )
-        if not clear_chunk_success:
-            return clear_chunk_success, clear_chunk_msg
+        
+        keys_to_delete = []
+        for member_bytes in members:
+            member_str = member_bytes.decode('utf-8')
+            member_doc = json.loads(member_str)
+            ix_str = member_doc.get('ix')
+            if not ix_str:
+                continue
+            
+            ix_doc = string_to_dict(ix_str.replace(COLON, ':'))
+            doc_key = self.get_document_key(ix_doc, list(ix_doc.keys()), table_name)
+            keys_to_delete.append(doc_key)
+
+        if keys_to_delete:
+            batch_size = 1000
+            for i in range(0, len(keys_to_delete), batch_size):
+                batch = keys_to_delete[i:i+batch_size]
+                self.client.delete(*batch)
+
+    except Exception as e:
+        return False, f"Failed to delete documents for {pipe}:\n{e}"
+
     try:
         self.drop_table(pipe.target, debug=debug)
     except Exception as e:
@@ -337,7 +363,7 @@ def drop_pipe(
     if 'valkey' not in pipe.parameters:
         return True, "Success"
 
-    pipe.parameters['valkey']['dtypes'] = {}
+    pipe._attributes['parameters']['valkey']['dtypes'] = {}
     if not pipe.temporary:
         edit_success, edit_msg = pipe.edit(debug=debug)
         if not edit_success:
@@ -558,11 +584,7 @@ def sync_pipe(
 
     valkey_dtypes = pipe.parameters.get('valkey', {}).get('dtypes', {})
     new_dtypes = {
-        str(key): (
-            str(val)
-            if not are_dtypes_equal(str(val), 'datetime')
-            else 'datetime64[ns, UTC]'
-        )
+        str(key): str(val)
         for key, val in df.dtypes.items()
         if str(key) not in valkey_dtypes
     }
@@ -571,19 +593,20 @@ def sync_pipe(
             try:
                 df[col] = df[col].astype(typ)
             except Exception:
+                import traceback
+                traceback.print_exc()
                 valkey_dtypes[col] = 'string'
                 new_dtypes[col] = 'string'
                 df[col] = df[col].astype('string')
 
     if new_dtypes and (not static or not valkey_dtypes):
         valkey_dtypes.update(new_dtypes)
-        if 'valkey' not in pipe.parameters:
-            pipe.parameters['valkey'] = {}
-        pipe.parameters['valkey']['dtypes'] = valkey_dtypes
-        if not pipe.temporary:
-            edit_success, edit_msg = pipe.edit(debug=debug)
-            if not edit_success:
-                return edit_success, edit_msg
+        update_success, update_msg = pipe.update_parameters(
+            {'valkey': {'dtypes': valkey_dtypes}},
+            debug=debug,
+        )
+        if not update_success:
+            return False, update_msg
 
     unseen_df, update_df, delta_df = (
         pipe.filter_existing(df, include_unchanged_columns=True, debug=debug)
@@ -781,7 +804,7 @@ def get_sync_time(
     """
     from meerschaum.utils.dtypes import are_dtypes_equal
     dt_col = pipe.columns.get('datetime', None)
-    dt_typ = pipe.dtypes.get(dt_col, 'datetime64[ns, UTC]')
+    dt_typ = pipe.dtypes.get(dt_col, 'datetime')
     if not dt_col:
         return None
 
@@ -789,14 +812,18 @@ def get_sync_time(
     table_name = self.quote_table(pipe.target)
     try:
         vals = (
-            self.client.zrevrange(table_name, 0, 0)
+            self.client.zrevrange(table_name, 0, 0, withscores=True)
             if newest
-            else self.client.zrange(table_name, 0, 0)
+            else self.client.zrange(table_name, 0, 0, withscores=True)
         )
         if not vals:
             return None
-        val = vals[0]
+        val = vals[0][0]
+        if isinstance(val, bytes):
+            val = val.decode('utf-8')
     except Exception:
+        import traceback
+        traceback.print_exc()
         return None
 
     doc = json.loads(val)

meerschaum/connectors/valkey/_plugins.py

@@ -127,7 +127,8 @@ def get_plugin_id(
     """
     Return a plugin's ID.
     """
-    return plugin.name
+    user_id = self.get_plugin_user_id(plugin, debug=debug)
+    return plugin.name if user_id is not None else None
 
 
 def get_plugin_version(
@@ -199,31 +200,6 @@ def get_plugin_attributes(
         return {}
 
 
-def get_plugins(
-    self,
-    user_id: Optional[int] = None,
-    search_term: Optional[str] = None,
-    debug: bool = False,
-    **kw: Any
-) -> List[str]:
-    """
-    Return a list of plugin names.
-    """
-    plugins_pipe = self.get_plugins_pipe()
-    params = {}
-    if user_id:
-        params['user_id'] = user_id
-
-    df = plugins_pipe.get_data(['plugin_name'], params=params, debug=debug)
-    docs = df.to_dict(orient='records')
-
-    return [
-        doc['plugin_name']
-        for doc in docs
-        if (plugin_name := doc['plugin_name']).startswith(search_term or '')
-    ]
-
-
 def delete_plugin(
     self,
     plugin: 'mrsm.core.Plugin',

meerschaum/core/Pipe/__init__.py

@@ -50,12 +50,16 @@ with correct credentials, as well as a network connection and valid permissions.
 """
 
 from __future__ import annotations
+
 import sys
 import copy
-from meerschaum.utils.typing import Optional, Dict, Any, Union, InstanceConnector, List
+
+import meerschaum as mrsm
+from meerschaum.utils.typing import Optional, Dict, Any, Union, List, InstanceConnector
 from meerschaum.utils.formatting._pipes import pipe_repr
 from meerschaum.config import get_config
 
+
 class Pipe:
     """
     Access Meerschaum pipes via Pipe objects.
@@ -89,6 +93,9 @@ class Pipe:
         get_data,
         get_backtrack_data,
         get_rowcount,
+        get_data,
+        get_doc,
+        get_value,
         _get_data_as_iterator,
         get_chunk_interval,
         get_chunk_bounds,
@@ -104,15 +111,20 @@ class Pipe:
         indexes,
         dtypes,
         autoincrement,
+        autotime,
         upsert,
         static,
         tzinfo,
         enforce,
         null_indices,
+        mixed_numerics,
         get_columns,
         get_columns_types,
         get_columns_indices,
         get_indices,
+        get_parameters,
+        get_dtypes,
+        update_parameters,
         tags,
         get_id,
         id,
@@ -123,6 +135,9 @@ class Pipe:
         target,
         _target_legacy,
         guess_datetime,
+        precision,
+        get_precision,
+        _invalidate_cache,
     )
     from ._show import show
     from ._edit import edit, edit_definition, update
@@ -133,11 +148,7 @@ class Pipe:
         filter_existing,
         _get_chunk_label,
         get_num_workers,
-        _persist_new_json_columns,
-        _persist_new_numeric_columns,
-        _persist_new_uuid_columns,
-        _persist_new_bytes_columns,
-        _persist_new_geometry_columns,
+        _persist_new_special_columns,
     )
     from ._verify import (
         verify,
@@ -168,9 +179,12 @@ class Pipe:
         temporary: bool = False,
         upsert: Optional[bool] = None,
         autoincrement: Optional[bool] = None,
+        autotime: Optional[bool] = None,
+        precision: Union[str, Dict[str, Union[str, int]], None] = None,
         static: Optional[bool] = None,
         enforce: Optional[bool] = None,
         null_indices: Optional[bool] = None,
+        mixed_numerics: Optional[bool] = None,
         mrsm_instance: Optional[Union[str, InstanceConnector]] = None,
         cache: bool = False,
         debug: bool = False,
@@ -226,6 +240,16 @@ class Pipe:
         autoincrement: Optional[bool], default None
             If `True`, set `autoincrement` in the parameters.
 
+        autotime: Optional[bool], default None
+            If `True`, set `autotime` in the parameters.
+
+        precision: Union[str, Dict[str, Union[str, int]], None], default None
+            If provided, set `precision` in the parameters.
+            This may be either a string (the precision unit) or a dictionary of in the form
+            `{'unit': <unit>, 'interval': <interval>}`.
+            Default is determined by the `datetime` column dtype
+            (e.g. `datetime64[us]` is `microsecond` precision).
+
         static: Optional[bool], default None
             If `True`, set `static` in the parameters.
 
@@ -237,6 +261,11 @@ class Pipe:
             Set to `False` if there will be no null values in the index columns.
             Defaults to `True`.
 
+        mixed_numerics: bool, default None
+            If `True`, integer columns will be converted to `numeric` when floats are synced.
+            Set to `False` to disable this behavior.
+            Defaults to `True`.
+
         temporary: bool, default False
             If `True`, prevent instance tables (pipes, users, plugins) from being created.
 
@@ -264,7 +293,7 @@ class Pipe:
         if location in ('[None]', 'None'):
             location = None
 
-        from meerschaum.config.static import STATIC_CONFIG
+        from meerschaum._internal.static import STATIC_CONFIG
         negation_prefix = STATIC_CONFIG['system']['fetch_pipes_keys']['negation_prefix']
         for k in (connector, metric, location, *(tags or [])):
             if str(k).startswith(negation_prefix):
@@ -291,11 +320,13 @@ class Pipe:
                 warn(f"The provided parameters are of invalid type '{type(parameters)}'.")
             self._attributes['parameters'] = {}
 
-        columns = columns or self._attributes.get('parameters', {}).get('columns', {})
-        if isinstance(columns, list):
+        columns = columns or self._attributes.get('parameters', {}).get('columns', None)
+        if isinstance(columns, (list, tuple)):
             columns = {str(col): str(col) for col in columns}
         if isinstance(columns, dict):
             self._attributes['parameters']['columns'] = columns
+        elif isinstance(columns, str) and 'Pipe(' in columns:
+            pass
         elif columns is not None:
             warn(f"The provided columns are of invalid type '{type(columns)}'.")
 
@@ -334,6 +365,14 @@ class Pipe:
         if isinstance(autoincrement, bool):
             self._attributes['parameters']['autoincrement'] = autoincrement
 
+        if isinstance(autotime, bool):
+            self._attributes['parameters']['autotime'] = autotime
+
+        if isinstance(precision, dict):
+            self._attributes['parameters']['precision'] = precision
+        elif isinstance(precision, str):
+            self._attributes['parameters']['precision'] = {'unit': precision}
+
         if isinstance(static, bool):
             self._attributes['parameters']['static'] = static
 
@@ -343,6 +382,9 @@ class Pipe:
         if isinstance(null_indices, bool):
             self._attributes['parameters']['null_indices'] = null_indices
 
+        if isinstance(mixed_numerics, bool):
+            self._attributes['parameters']['mixed_numerics'] = mixed_numerics
+
         ### NOTE: The parameters dictionary is {} by default.
         ###       A Pipe may be registered without parameters, then edited,
         ###       or a Pipe may be registered with parameters set in-memory first.
@@ -383,9 +425,7 @@ class Pipe:
     @property
     def instance_connector(self) -> Union[InstanceConnector, None]:
         """
-        The connector to where this pipe resides.
-        May either be of type `meerschaum.connectors.sql.SQLConnector` or
-        `meerschaum.connectors.api.APIConnector`.
+        The instance connector on which this pipe resides.
         """
         if '_instance_connector' not in self.__dict__:
             from meerschaum.connectors.parse import parse_instance_keys
@@ -397,7 +437,7 @@ class Pipe:
         return self._instance_connector
 
     @property
-    def connector(self) -> Union[meerschaum.connectors.Connector, None]:
+    def connector(self) -> Union['Connector', None]:
         """
         The connector to the data source.
         """
@@ -417,7 +457,7 @@ class Pipe:
         return self._connector
 
     @property
-    def cache_connector(self) -> Union[meerschaum.connectors.sql.SQLConnector, None]:
+    def cache_connector(self) -> Union['Connector', None]:
         """
         If the pipe was created with `cache=True`, return the connector to the pipe's
         SQLite database for caching.
@@ -438,7 +478,7 @@ class Pipe:
         return self._cache_connector
 
     @property
-    def cache_pipe(self) -> Union['meerschaum.Pipe', None]:
+    def cache_pipe(self) -> Union[mrsm.Pipe, None]:
         """
         If the pipe was created with `cache=True`, return another `meerschaum.Pipe` used to
         manage the local data.
@@ -470,10 +510,10 @@ class Pipe:
                 self.instance_keys,
                 (self.connector_keys + '_' + self.metric_key + '_cache'),
                 self.location_key,
-                mrsm_instance = self.cache_connector,
-                parameters = _parameters,
-                cache = False,
-                temporary = True,
+                mrsm_instance=self.cache_connector,
+                parameters=_parameters,
+                cache=False,
+                temporary=True,
             )
 
         return self._cache_pipe