meerschaum 2.7.6__py3-none-any.whl → 2.7.8__py3-none-any.whl

meerschaum/connectors/sql/_plugins.py

@@ -21,9 +21,8 @@ def register_plugin(
     **kw: Any
 ) -> SuccessTuple:
     """Register a new plugin to the plugins table."""
-    from meerschaum.utils.warnings import warn, error
     from meerschaum.utils.packages import attempt_import
-    sqlalchemy = attempt_import('sqlalchemy')
+    sqlalchemy = attempt_import('sqlalchemy', lazy=False)
     from meerschaum.utils.sql import json_flavors
     from meerschaum.connectors.sql.tables import get_tables
     plugins_tbl = get_tables(mrsm_instance=self, debug=debug)['plugins']
@@ -85,7 +84,7 @@ def get_plugin_id(
     from meerschaum.connectors.sql.tables import get_tables
     plugins_tbl = get_tables(mrsm_instance=self, debug=debug)['plugins']
     from meerschaum.utils.packages import attempt_import
-    sqlalchemy = attempt_import('sqlalchemy')
+    sqlalchemy = attempt_import('sqlalchemy', lazy=False)
 
     query = (
         sqlalchemy
@@ -95,9 +94,10 @@ def get_plugin_id(
     
     try:
         return int(self.value(query, debug=debug))
-    except Exception as e:
+    except Exception:
         return None
 
+
 def get_plugin_version(
     self,
     plugin: 'mrsm.core.Plugin',
@@ -110,7 +110,7 @@ def get_plugin_version(
     from meerschaum.connectors.sql.tables import get_tables
     plugins_tbl = get_tables(mrsm_instance=self, debug=debug)['plugins']
     from meerschaum.utils.packages import attempt_import
-    sqlalchemy = attempt_import('sqlalchemy')
+    sqlalchemy = attempt_import('sqlalchemy', lazy=False)
     query = sqlalchemy.select(plugins_tbl.c.version).where(plugins_tbl.c.plugin_name == plugin.name)
     return self.value(query, debug=debug)
 
@@ -126,7 +126,7 @@ def get_plugin_user_id(
     from meerschaum.connectors.sql.tables import get_tables
     plugins_tbl = get_tables(mrsm_instance=self, debug=debug)['plugins']
     from meerschaum.utils.packages import attempt_import
-    sqlalchemy = attempt_import('sqlalchemy')
+    sqlalchemy = attempt_import('sqlalchemy', lazy=False)
 
     query = (
         sqlalchemy
@@ -136,7 +136,7 @@ def get_plugin_user_id(
 
     try:
         return int(self.value(query, debug=debug))
-    except Exception as e:
+    except Exception:
         return None
 
 def get_plugin_username(
@@ -152,7 +152,7 @@ def get_plugin_username(
     plugins_tbl = get_tables(mrsm_instance=self, debug=debug)['plugins']
     users = get_tables(mrsm_instance=self, debug=debug)['users']
     from meerschaum.utils.packages import attempt_import
-    sqlalchemy = attempt_import('sqlalchemy')
+    sqlalchemy = attempt_import('sqlalchemy', lazy=False)
 
     query = (
         sqlalchemy.select(users.c.username)
@@ -177,7 +177,7 @@ def get_plugin_attributes(
     from meerschaum.connectors.sql.tables import get_tables
     plugins_tbl = get_tables(mrsm_instance=self, debug=debug)['plugins']
     from meerschaum.utils.packages import attempt_import
-    sqlalchemy = attempt_import('sqlalchemy')
+    sqlalchemy = attempt_import('sqlalchemy', lazy=False)
 
     query = (
         sqlalchemy
@@ -219,7 +219,7 @@ def get_plugins(
     from meerschaum.connectors.sql.tables import get_tables
     plugins_tbl = get_tables(mrsm_instance=self, debug=debug)['plugins']
     from meerschaum.utils.packages import attempt_import
-    sqlalchemy = attempt_import('sqlalchemy')
+    sqlalchemy = attempt_import('sqlalchemy', lazy=False)
 
     query = sqlalchemy.select(plugins_tbl.c.plugin_name)
     if user_id is not None:
@@ -246,9 +246,8 @@ def delete_plugin(
     **kw: Any
 ) -> SuccessTuple:
     """Delete a plugin from the plugins table."""
-    from meerschaum.utils.warnings import warn, error
     from meerschaum.utils.packages import attempt_import
-    sqlalchemy = attempt_import('sqlalchemy')
+    sqlalchemy = attempt_import('sqlalchemy', lazy=False)
     from meerschaum.connectors.sql.tables import get_tables
     plugins_tbl = get_tables(mrsm_instance=self, debug=debug)['plugins']
 
@@ -256,10 +255,6 @@ def delete_plugin(
     if plugin_id is None:
         return True, f"Plugin '{plugin}' was not registered."
 
-    bind_variables = {
-        'plugin_id' : plugin_id,
-    }
-
     query = sqlalchemy.delete(plugins_tbl).where(plugins_tbl.c.plugin_id == plugin_id)
     result = self.exec(query, debug=debug)
     if result is None:

meerschaum/connectors/sql/_sql.py

@@ -126,7 +126,7 @@ def read(
         return []
     from meerschaum.utils.sql import sql_item_name, truncate_item_name
     from meerschaum.utils.dtypes import are_dtypes_equal, coerce_timezone
-    from meerschaum.utils.dtypes.sql import NUMERIC_PRECISION_FLAVORS, TIMEZONE_NAIVE_FLAVORS
+    from meerschaum.utils.dtypes.sql import TIMEZONE_NAIVE_FLAVORS
     from meerschaum.utils.packages import attempt_import, import_pandas
     from meerschaum.utils.pool import get_pool
     from meerschaum.utils.dataframe import chunksize_to_npartitions, get_numeric_cols
@@ -154,7 +154,7 @@ def read(
             dtype[col] = 'datetime64[ns]'
 
     pool = get_pool(workers=workers)
-    sqlalchemy = attempt_import("sqlalchemy")
+    sqlalchemy = attempt_import("sqlalchemy", lazy=False)
     default_chunksize = self._sys_config.get('chunksize', None)
     chunksize = chunksize if chunksize != -1 else default_chunksize
     if chunksize is None and as_iterator:
@@ -443,7 +443,6 @@ def value(
 
     """
     from meerschaum.utils.packages import attempt_import
-    sqlalchemy = attempt_import('sqlalchemy')
     if self.flavor == 'duckdb':
         use_pandas = True
     if use_pandas:
@@ -455,9 +454,6 @@ def value(
     _close = kw.get('close', True)
     _commit = kw.get('commit', (self.flavor != 'mssql'))
 
-    #  _close = True
-    #  _commit = True
-
     try:
         result, connection = self.exec(
             query,
@@ -556,7 +552,7 @@ def exec(
         )
 
     from meerschaum.utils.packages import attempt_import
-    sqlalchemy = attempt_import("sqlalchemy")
+    sqlalchemy = attempt_import("sqlalchemy", lazy=False)
     if debug:
         dprint(f"[{self}] Executing query:\n{query}")
 
@@ -659,7 +655,7 @@ def exec_queries(
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.debug import dprint
     from meerschaum.utils.packages import attempt_import
-    sqlalchemy, sqlalchemy_orm = attempt_import('sqlalchemy', 'sqlalchemy.orm')
+    sqlalchemy, sqlalchemy_orm = attempt_import('sqlalchemy', 'sqlalchemy.orm', lazy=False)
     session = sqlalchemy_orm.Session(self.engine)
 
     result = None
@@ -806,26 +802,37 @@ def to_sql(
     )
     from meerschaum.utils.dtypes import (
         are_dtypes_equal,
-        quantize_decimal,
         coerce_timezone,
         encode_bytes_for_bytea,
         serialize_bytes,
+        serialize_decimal,
+        json_serialize_value,
     )
     from meerschaum.utils.dtypes.sql import (
-        NUMERIC_PRECISION_FLAVORS,
-        NUMERIC_AS_TEXT_FLAVORS,
         PD_TO_SQLALCHEMY_DTYPES_FLAVORS,
         get_db_type_from_pd_type,
+        get_pd_type_from_db_type,
+        get_numeric_precision_scale,
     )
     from meerschaum.utils.misc import interval_str
     from meerschaum.connectors.sql._create_engine import flavor_configs
     from meerschaum.utils.packages import attempt_import, import_pandas
-    sqlalchemy = attempt_import('sqlalchemy', debug=debug)
+    sqlalchemy = attempt_import('sqlalchemy', debug=debug, lazy=False)
     pd = import_pandas()
     is_dask = 'dask' in df.__module__
 
     bytes_cols = get_bytes_cols(df)
     numeric_cols = get_numeric_cols(df)
+    numeric_cols_dtypes = {
+        col: typ
+        for col, typ in kw.get('dtype', {}).items()
+        if (
+            col in df.columns
+            and 'numeric' in str(typ).lower()
+        )
+        
+    }
+    numeric_cols.extend([col for col in numeric_cols_dtypes if col not in numeric_cols])
 
     enable_bulk_insert = mrsm.get_config(
         'system', 'connectors', 'sql', 'bulk_insert'
@@ -858,12 +865,24 @@ def to_sql(
         for col in bytes_cols:
             df[col] = df[col].apply(bytes_serializer)
 
-    if self.flavor in NUMERIC_AS_TEXT_FLAVORS:
-        if safe_copy and not copied:
-            df = df.copy()
-            copied = True
-        for col in numeric_cols:
-            df[col] = df[col].astype(str)
+    ### Check for numeric columns.
+    for col in numeric_cols:
+        typ = numeric_cols_dtypes.get(col, None)
+
+        precision, scale = (
+            (typ.precision, typ.scale)
+            if hasattr(typ, 'precision')
+            else get_numeric_precision_scale(self.flavor)
+        )
+
+        df[col] = df[col].apply(
+            functools.partial(
+                serialize_decimal,
+                quantize=True,
+                precision=precision,
+                scale=scale,
+            )
+        )
 
     stats['method'] = method.__name__ if hasattr(method, '__name__') else str(method)
 
@@ -893,7 +912,7 @@ def to_sql(
     if name != truncated_name:
         warn(
             f"Table '{name}' is too long for '{self.flavor}',"
-            + f" will instead create the table '{truncated_name}'."
+            f" will instead create the table '{truncated_name}'."
         )
 
     ### filter out non-pandas args
@@ -961,24 +980,11 @@ def to_sql(
     ### Check for JSON columns.
     if self.flavor not in json_flavors:
         json_cols = get_json_cols(df)
-        if json_cols:
-            for col in json_cols:
-                df[col] = df[col].apply(
-                    (
-                        lambda x: json.dumps(x, default=str, sort_keys=True)
-                        if not isinstance(x, Hashable)
-                        else x
-                    )
-                )
-
-    ### Check for numeric columns.
-    numeric_scale, numeric_precision = NUMERIC_PRECISION_FLAVORS.get(self.flavor, (None, None))
-    if numeric_precision is not None and numeric_scale is not None:
-        for col in numeric_cols:
+        for col in json_cols:
             df[col] = df[col].apply(
-                lambda x: (
-                    quantize_decimal(x, numeric_scale, numeric_precision)
-                    if isinstance(x, Decimal)
+                (
+                    lambda x: json.dumps(x, default=json_serialize_value, sort_keys=True)
+                    if not isinstance(x, Hashable)
                     else x
                 )
             )
@@ -1055,16 +1061,20 @@ def psql_insert_copy(
 
     from meerschaum.utils.sql import sql_item_name
     from meerschaum.utils.warnings import dprint
+    from meerschaum.utils.dtypes import json_serialize_value
 
     ### NOTE: PostgreSQL doesn't support NUL chars in text, so they're removed from strings.
     data_iter = (
         (
             (
                 (
-                    json.dumps(item).replace('\0', '').replace('\\u0000', '')
+                    json.dumps(
+                        item,
+                        default=json_serialize_value,
+                    ).replace('\0', '').replace('\\u0000', '')
                     if isinstance(item, (dict, list))
                     else (
-                        item
+                        json_serialize_value(item, default_to_str=False)
                         if not isinstance(item, str)
                         else item.replace('\0', '').replace('\\u0000', '')
                     )
@@ -1123,6 +1133,7 @@ def mssql_insert_json(
     """
     import json
     from meerschaum.utils.sql import sql_item_name
+    from meerschaum.utils.dtypes import json_serialize_value
     from meerschaum.utils.dtypes.sql import get_pd_type_from_db_type, get_db_type_from_pd_type
     from meerschaum.utils.warnings import dprint
     table_name = sql_item_name(table.name, 'mssql', table.schema)
@@ -1131,6 +1142,15 @@ def mssql_insert_json(
             str(column.name): get_pd_type_from_db_type(str(column.type))
             for column in table.table.columns
         }
+        numeric_cols_types = {
+            col: table.table.columns[col].type
+            for col, typ in pd_types.items()
+            if typ.startswith('numeric') and col in keys
+        }
+        pd_types.update({
+            col: f'numeric[{typ.precision},{typ.scale}]'
+            for col, typ in numeric_cols_types.items()
+        })
         cols_types = {
             col: get_db_type_from_pd_type(typ, 'mssql')
             for col, typ in pd_types.items()
@@ -1155,7 +1175,8 @@ def mssql_insert_json(
     if debug:
         dprint(sql)
 
-    conn.exec_driver_sql(sql, (json.dumps(json_data, default=str),))
+    serialized_data = json.dumps(json_data, default=json_serialize_value)
+    conn.exec_driver_sql(sql, (serialized_data,))
 
 
 def format_sql_query_for_dask(query: str) -> 'sqlalchemy.sql.selectable.Select':

meerschaum/connectors/sql/_uri.py

@@ -13,14 +13,14 @@ from meerschaum.utils.packages import attempt_import
 
 @classmethod
 def from_uri(
-        cls,
-        uri: str,
-        label: Optional[str] = None,
-        as_dict: bool = False,
-    ) -> Union[
-        'meerschaum.connectors.SQLConnector',
-        Dict[str, Union[str, int]],
-    ]:
+    cls,
+    uri: str,
+    label: Optional[str] = None,
+    as_dict: bool = False,
+) -> Union[
+    'meerschaum.connectors.SQLConnector',
+    Dict[str, Union[str, int]],
+]:
     """
     Create a new SQLConnector from a URI string.
 
@@ -97,7 +97,7 @@ def parse_uri(uri: str) -> Dict[str, Any]:
     >>> 
     """
     from urllib.parse import parse_qs, urlparse
-    sqlalchemy = attempt_import('sqlalchemy')
+    sqlalchemy = attempt_import('sqlalchemy', lazy=False)
     parser = sqlalchemy.engine.url.make_url
     params = parser(uri).translate_connect_args()
     params['flavor'] = uri.split(':')[0].split('+')[0]

meerschaum/connectors/sql/_users.py

@@ -19,10 +19,9 @@ def register_user(
     **kw: Any
 ) -> SuccessTuple:
     """Register a new user."""
-    from meerschaum.utils.warnings import warn, error, info
     from meerschaum.utils.packages import attempt_import
     from meerschaum.utils.sql import json_flavors
-    sqlalchemy = attempt_import('sqlalchemy')
+    sqlalchemy = attempt_import('sqlalchemy', lazy=False)
 
     valid_tuple = valid_username(user.username)
     if not valid_tuple[0]:
@@ -103,9 +102,8 @@ def edit_user(
 ) -> SuccessTuple:
     """Update an existing user's metadata."""
     from meerschaum.utils.packages import attempt_import
-    sqlalchemy = attempt_import('sqlalchemy')
+    sqlalchemy = attempt_import('sqlalchemy', lazy=False)
     from meerschaum.connectors.sql.tables import get_tables
-    from meerschaum.utils.sql import json_flavors
     users_tbl = get_tables(mrsm_instance=self, debug=debug)['users']
 
     user_id = user.user_id if user.user_id is not None else self.get_user_id(user, debug=debug)
@@ -158,7 +156,7 @@ def get_user_id(
     """If a user is registered, return the `user_id`."""
     ### ensure users table exists
     from meerschaum.utils.packages import attempt_import
-    sqlalchemy = attempt_import('sqlalchemy')
+    sqlalchemy = attempt_import('sqlalchemy', lazy=False)
     from meerschaum.connectors.sql.tables import get_tables
     users_tbl = get_tables(mrsm_instance=self, debug=debug)['users']
 
@@ -183,7 +181,7 @@ def get_user_attributes(
     ### ensure users table exists
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.packages import attempt_import
-    sqlalchemy = attempt_import('sqlalchemy')
+    sqlalchemy = attempt_import('sqlalchemy', lazy=False)
     from meerschaum.connectors.sql.tables import get_tables
     users_tbl = get_tables(mrsm_instance=self, debug=debug)['users']
 
@@ -199,14 +197,14 @@ def get_user_attributes(
         try:
             result = dict(result)
             _parsed = True
-        except Exception as e:
+        except Exception:
             _parsed = False
         if not _parsed:
             try:
                 import json
                 result = json.loads(result)
                 _parsed = True
-            except Exception as e:
+            except Exception:
                 _parsed = False
         if not _parsed:
             warn(f"Received unexpected type for attributes: {result}")
@@ -223,7 +221,7 @@ def delete_user(
     users_tbl = get_tables(mrsm_instance=self, debug=debug)['users']
     plugins = get_tables(mrsm_instance=self, debug=debug)['plugins']
     from meerschaum.utils.packages import attempt_import
-    sqlalchemy = attempt_import('sqlalchemy')
+    sqlalchemy = attempt_import('sqlalchemy', lazy=False)
 
     user_id = user.user_id if user.user_id is not None else self.get_user_id(user, debug=debug)
 
@@ -256,7 +254,7 @@ def get_users(
     from meerschaum.connectors.sql.tables import get_tables
     users_tbl = get_tables(mrsm_instance=self, debug=debug)['users']
     from meerschaum.utils.packages import attempt_import
-    sqlalchemy = attempt_import('sqlalchemy')
+    sqlalchemy = attempt_import('sqlalchemy', lazy=False)
 
     query = sqlalchemy.select(users_tbl.c.username)
 
@@ -277,7 +275,7 @@ def get_user_password_hash(
     from meerschaum.connectors.sql.tables import get_tables
     users_tbl = get_tables(mrsm_instance=self, debug=debug)['users']
     from meerschaum.utils.packages import attempt_import
-    sqlalchemy = attempt_import('sqlalchemy')
+    sqlalchemy = attempt_import('sqlalchemy', lazy=False)
 
     if user.user_id is not None:
         user_id = user.user_id
@@ -308,7 +306,7 @@ def get_user_type(
     from meerschaum.connectors.sql.tables import get_tables
     users_tbl = get_tables(mrsm_instance=self, debug=debug)['users']
     from meerschaum.utils.packages import attempt_import
-    sqlalchemy = attempt_import('sqlalchemy')
+    sqlalchemy = attempt_import('sqlalchemy', lazy=False)
 
     user_id = user.user_id if user.user_id is not None else self.get_user_id(user, debug=debug)
 

meerschaum/connectors/sql/tables/__init__.py

@@ -17,10 +17,10 @@ _sequence_flavors = {'duckdb', 'oracle'}
 _skip_index_names_flavors = {'mssql',}
 
 def get_tables(
-        mrsm_instance: Optional[Union[str, InstanceConnector]] = None,
-        create: bool = True,
-        debug: Optional[bool] = None
-    ) -> Union[Dict[str, 'sqlalchemy.Table'], bool]:
+    mrsm_instance: Optional[Union[str, InstanceConnector]] = None,
+    create: bool = True,
+    debug: Optional[bool] = None
+) -> Union[Dict[str, 'sqlalchemy.Table'], bool]:
     """
     Create tables on the database and return the `sqlalchemy` tables.
 
@@ -51,7 +51,7 @@ def get_tables(
     sqlalchemy, sqlalchemy_dialects_postgresql = attempt_import(
         'sqlalchemy',
         'sqlalchemy.dialects.postgresql',
-        lazy = False
+        lazy=False,
     )
     if not sqlalchemy:
         error(f"Failed to import sqlalchemy. Is sqlalchemy installed?")
@@ -205,13 +205,12 @@ def get_tables(
 
 
 def create_tables(
-        conn: 'meerschaum.connectors.SQLConnector',
-        tables: Optional[Dict[str, 'sqlalchemy.Table']] = None,
-    ) -> bool:
+    conn: 'meerschaum.connectors.SQLConnector',
+    tables: Optional[Dict[str, 'sqlalchemy.Table']] = None,
+) -> bool:
     """
     Create the tables on the database.
     """
-    from meerschaum.utils.sql import get_rename_table_queries, table_exists
     _tables = tables if tables is not None else get_tables(conn)
 
     try:
@@ -225,10 +224,10 @@ def create_tables(
 
 
 def create_schemas(
-        conn: 'meerschaum.connectors.SQLConnector',
-        schemas: List[str],
-        debug: bool = False,
-    ) -> bool:
+    conn: 'meerschaum.connectors.SQLConnector',
+    schemas: List[str],
+    debug: bool = False,
+) -> bool:
     """
     Create the internal Meerschaum schema on the database.
     """
@@ -238,7 +237,7 @@ def create_schemas(
     if conn.flavor in NO_SCHEMA_FLAVORS:
         return True
 
-    sqlalchemy_schema = attempt_import('sqlalchemy.schema')
+    _ = attempt_import('sqlalchemy.schema', lazy=False)
     successes = {}
     skip_if_not_exists = conn.flavor in SKIP_IF_EXISTS_FLAVORS
     if_not_exists_str = ("IF NOT EXISTS " if not skip_if_not_exists else "")

meerschaum/connectors/valkey/_ValkeyConnector.py

@@ -239,7 +239,7 @@ class ValkeyConnector(Connector):
         -------
         The current index counter value (how many docs have been pushed).
         """
-        from meerschaum.utils.misc import json_serialize_datetime
+        from meerschaum.utils.dtypes import json_serialize_value
         table_name = self.quote_table(table)
         datetime_column_key = self.get_datetime_column_key(table)
         remote_datetime_column = self.get(datetime_column_key)
@@ -269,7 +269,7 @@ class ValkeyConnector(Connector):
             ) if datetime_column else None
             doc_str = json.dumps(
                 doc,
-                default=(lambda x: json_serialize_datetime(x) if hasattr(x, 'tzinfo') else str(x)),
+                default=json_serialize_value,
                 separators=(',', ':'),
                 sort_keys=True,
             )

meerschaum/core/Pipe/__init__.py

@@ -107,6 +107,7 @@ class Pipe:
         static,
         tzinfo,
         enforce,
+        null_indices,
         get_columns,
         get_columns_types,
         get_columns_indices,
@@ -141,7 +142,8 @@ class Pipe:
         get_bound_time,
     )
     from ._delete import delete
-    from ._drop import drop
+    from ._drop import drop, drop_indices
+    from ._index import create_indices
     from ._clear import clear
     from ._deduplicate import deduplicate
     from ._bootstrap import bootstrap
@@ -165,6 +167,7 @@ class Pipe:
         autoincrement: Optional[bool] = None,
         static: Optional[bool] = None,
         enforce: Optional[bool] = None,
+        null_indices: Optional[bool] = None,
         mrsm_instance: Optional[Union[str, InstanceConnector]] = None,
         cache: bool = False,
         debug: bool = False,
@@ -223,10 +226,14 @@ class Pipe:
         static: Optional[bool], default None
             If `True`, set `static` in the parameters.
 
-        enforce: Optionanl[bool], default None
+        enforce: Optional[bool], default None
             If `False`, skip data type enforcement.
             Default behavior is `True`.
 
+        null_indices: Optional[bool], default None
+            Set to `False` if there will be no null values in the index columns.
+            Defaults to `True`.
+
         temporary: bool, default False
             If `True`, prevent instance tables (pipes, users, plugins) from being created.
 
@@ -330,6 +337,9 @@ class Pipe:
         if isinstance(enforce, bool):
             self._attributes['parameters']['enforce'] = enforce
 
+        if isinstance(null_indices, bool):
+            self._attributes['parameters']['null_indices'] = null_indices
+
         ### NOTE: The parameters dictionary is {} by default.
         ###       A Pipe may be registered without parameters, then edited,
         ###       or a Pipe may be registered with parameters set in-memory first.

meerschaum/core/Pipe/_attributes.py

@@ -11,7 +11,7 @@ from __future__ import annotations
 from datetime import timezone
 
 import meerschaum as mrsm
-from meerschaum.utils.typing import Tuple, Dict, SuccessTuple, Any, Union, Optional, List
+from meerschaum.utils.typing import Tuple, Dict, Any, Union, Optional, List
 from meerschaum.utils.warnings import warn
 
 
@@ -313,6 +313,25 @@ def enforce(self, _enforce: bool) -> None:
     self.parameters['enforce'] = _enforce
 
 
+@property
+def null_indices(self) -> bool:
+    """
+    Return the `null_indices` parameter for the pipe.
+    """
+    if 'null_indices' not in self.parameters:
+        self.parameters['null_indices'] = True
+
+    return self.parameters['null_indices']
+
+
+@null_indices.setter
+def null_indices(self, _null_indices: bool) -> None:
+    """
+    Set the `null_indices` parameter for the pipe.
+    """
+    self.parameters['null_indices'] = _null_indices
+
+
 def get_columns(self, *args: str, error: bool = False) -> Union[str, Tuple[str]]:
     """
     Check if the requested columns are defined.
@@ -469,7 +488,7 @@ def get_columns_indices(
 
     self.__dict__['_columns_indices'] = _columns_indices
     self.__dict__['_columns_indices_timestamp'] = now
-    return _columns_indices or {}
+    return {k: v for k, v in _columns_indices.items() if k and v} or {}
 
 
 def get_id(self, **kw: Any) -> Union[int, None]:
@@ -711,42 +730,17 @@ def guess_datetime(self) -> Union[str, None]:
 
 def get_indices(self) -> Dict[str, str]:
     """
-    Return a dictionary mapping index keys to their names on the database.
+    Return a dictionary mapping index keys to their names in the database.
 
     Returns
     -------
-    A dictionary of index keys to column names.
-    """
-    _parameters = self.parameters
-    _index_template = _parameters.get('index_template', "IX_{target}_{column_names}")
-    _indices = self.indices
-    _target = self.target
-    _column_names = {
-        ix: (
-            '_'.join(cols)
-            if isinstance(cols, (list, tuple))
-            else str(cols)
-        )
-        for ix, cols in _indices.items()
-        if cols
-    }
-    _index_names = {
-        ix: _index_template.format(
-            target=_target,
-            column_names=column_names,
-            connector_keys=self.connector_keys,
-            metric_key=self.connector_key,
-            location_key=self.location_key,
-        )
-        for ix, column_names in _column_names.items()
-    }
-    ### NOTE: Skip any duplicate indices.
-    seen_index_names = {}
-    for ix, index_name in _index_names.items():
-        if index_name in seen_index_names:
-            continue
-        seen_index_names[index_name] = ix
-    return {
-        ix: index_name
-        for index_name, ix in seen_index_names.items()
-    }
+    A dictionary of index keys to index names.
+    """
+    from meerschaum.connectors import get_connector_plugin
+    with mrsm.Venv(get_connector_plugin(self.instance_connector)):
+        if hasattr(self.instance_connector, 'get_pipe_index_names'):
+            result = self.instance_connector.get_pipe_index_names(self)
+        else:
+            result = {}
+    
+    return result