meerschaum 2.7.6__py3-none-any.whl → 2.7.7__py3-none-any.whl

meerschaum/connectors/sql/_fetch.py

@@ -148,7 +148,6 @@ def get_pipe_metadef(
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.sql import sql_item_name, dateadd_str, build_where
     from meerschaum.utils.dtypes.sql import get_db_type_from_pd_type
-    from meerschaum.utils.misc import is_int
     from meerschaum.config import get_config
 
     dt_col = pipe.columns.get('datetime', None)
@@ -191,7 +190,7 @@ def get_pipe_metadef(
         else begin
     )
 
-    if begin and end and begin >= end:
+    if begin not in (None, '') and end is not None and begin >= end:
         begin = None
 
     if dt_name:
@@ -203,7 +202,7 @@ def get_pipe_metadef(
                 begin=begin,
                 db_type=db_dt_typ,
             )
-            if begin
+            if begin not in ('', None)
             else None
         )
         end_da = (
@@ -214,7 +213,7 @@ def get_pipe_metadef(
                 begin=end,
                 db_type=db_dt_typ,
             )
-            if end
+            if end is not None
             else None
         )
 
@@ -228,11 +227,7 @@ def get_pipe_metadef(
 
     has_where = 'where' in meta_def.lower()[meta_def.lower().rfind('definition'):]
     if dt_name and (begin_da or end_da):
-        definition_dt_name = (
-            dateadd_str(self.flavor, 'minute', 0, f"{definition_name}.{dt_name}", db_type=db_dt_typ)
-            if not is_int((begin_da or end_da))
-            else f"{definition_name}.{dt_name}"
-        )
+        definition_dt_name = f"{definition_name}.{dt_name}"
         meta_def += "\n" + ("AND" if has_where else "WHERE") + " "
         has_where = True
         if begin_da:

meerschaum/connectors/sql/_instance.py

@@ -25,7 +25,7 @@ def _log_temporary_tables_creation(
     """
     from meerschaum.utils.misc import items_str
     from meerschaum.connectors.sql.tables import get_tables
-    sqlalchemy = mrsm.attempt_import('sqlalchemy')
+    sqlalchemy = mrsm.attempt_import('sqlalchemy', lazy=False)
     temp_tables_table = get_tables(
         mrsm_instance=self,
         create=create,
@@ -86,7 +86,7 @@ def _drop_temporary_tables(self, debug: bool = False) -> SuccessTuple:
     """
     from meerschaum.utils.misc import items_str
     from meerschaum.connectors.sql.tables import get_tables
-    sqlalchemy = mrsm.attempt_import('sqlalchemy')
+    sqlalchemy = mrsm.attempt_import('sqlalchemy', lazy=False)
     temp_tables_table = get_tables(
         mrsm_instance=self,
         create=False,
@@ -150,7 +150,7 @@ def _drop_old_temporary_tables(
     """
     from meerschaum.config import get_config
     from meerschaum.connectors.sql.tables import get_tables
-    sqlalchemy = mrsm.attempt_import('sqlalchemy')
+    sqlalchemy = mrsm.attempt_import('sqlalchemy', lazy=False)
     temp_tables_table = get_tables(mrsm_instance=self, create=False, debug=debug)['temp_tables']
     last_check = getattr(self, '_stale_temporary_tables_check_timestamp', 0)
     now_ts = time.perf_counter()

meerschaum/connectors/sql/_pipes.py

@@ -55,7 +55,7 @@ def register_pipe(
         parameters = {}
 
     import json
-    sqlalchemy = attempt_import('sqlalchemy')
+    sqlalchemy = attempt_import('sqlalchemy', lazy=False)
     values = {
         'connector_keys' : pipe.connector_keys,
         'metric_key'     : pipe.metric_key,
@@ -118,7 +118,7 @@ def edit_pipe(
     pipes_tbl = get_tables(mrsm_instance=self, create=(not pipe.temporary), debug=debug)['pipes']
 
     import json
-    sqlalchemy = attempt_import('sqlalchemy')
+    sqlalchemy = attempt_import('sqlalchemy', lazy=False)
 
     values = {
         'parameters': (
@@ -176,7 +176,10 @@ def fetch_pipes_keys(
     from meerschaum.config.static import STATIC_CONFIG
     import json
     from copy import deepcopy
-    sqlalchemy, sqlalchemy_sql_functions = attempt_import('sqlalchemy', 'sqlalchemy.sql.functions')
+    sqlalchemy, sqlalchemy_sql_functions = attempt_import(
+        'sqlalchemy',
+        'sqlalchemy.sql.functions', lazy=False,
+    )
     coalesce = sqlalchemy_sql_functions.coalesce
 
     if connector_keys is None:
@@ -246,7 +249,7 @@ def fetch_pipes_keys(
 
     q = sqlalchemy.select(*select_cols).where(sqlalchemy.and_(True, *_where))
     for c, vals in cols.items():
-        if not isinstance(vals, (list, tuple)) or not vals or not c in pipes_tbl.c:
+        if not isinstance(vals, (list, tuple)) or not vals or c not in pipes_tbl.c:
             continue
         _in_vals, _ex_vals = separate_negation_values(vals)
         q = q.where(coalesce(pipes_tbl.c[c], 'None').in_(_in_vals)) if _in_vals else q
@@ -306,9 +309,28 @@ def fetch_pipes_keys(
     return [(row[0], row[1], row[2]) for row in rows]
 
 
+def create_pipe_indices(
+    self,
+    pipe: mrsm.Pipe,
+    columns: Optional[List[str]] = None,
+    debug: bool = False,
+) -> SuccessTuple:
+    """
+    Create a pipe's indices.
+    """
+    success = self.create_indices(pipe, columns=columns, debug=debug)
+    msg = (
+        "Success"
+        if success
+        else f"Failed to create indices for {pipe}."
+    )
+    return success, msg
+
+
 def create_indices(
     self,
     pipe: mrsm.Pipe,
+    columns: Optional[List[str]] = None,
     indices: Optional[List[str]] = None,
     debug: bool = False
 ) -> bool:
@@ -318,29 +340,51 @@ def create_indices(
     from meerschaum.utils.debug import dprint
     if debug:
         dprint(f"Creating indices for {pipe}...")
+
     if not pipe.indices:
         warn(f"{pipe} has no index columns; skipping index creation.", stack=False)
         return True
 
+    cols_to_include = set((columns or []) + (indices or [])) or None
+
     _ = pipe.__dict__.pop('_columns_indices', None)
     ix_queries = {
-        ix: queries
-        for ix, queries in self.get_create_index_queries(pipe, debug=debug).items()
-        if indices is None or ix in indices
+        col: queries
+        for col, queries in self.get_create_index_queries(pipe, debug=debug).items()
+        if cols_to_include is None or col in cols_to_include
     }
     success = True
-    for ix, queries in ix_queries.items():
+    for col, queries in ix_queries.items():
         ix_success = all(self.exec_queries(queries, debug=debug, silent=False))
         success = success and ix_success
         if not ix_success:
-            warn(f"Failed to create index on column: {ix}")
+            warn(f"Failed to create index on column: {col}")
 
     return success
 
 
+def drop_pipe_indices(
+    self,
+    pipe: mrsm.Pipe,
+    columns: Optional[List[str]] = None,
+    debug: bool = False,
+) -> SuccessTuple:
+    """
+    Drop a pipe's indices.
+    """
+    success = self.drop_indices(pipe, columns=columns, debug=debug)
+    msg = (
+        "Success"
+        if success
+        else f"Failed to drop indices for {pipe}."
+    )
+    return success, msg
+
+
 def drop_indices(
     self,
     pipe: mrsm.Pipe,
+    columns: Optional[List[str]] = None,
     indices: Optional[List[str]] = None,
     debug: bool = False
 ) -> bool:
@@ -350,24 +394,81 @@ def drop_indices(
     from meerschaum.utils.debug import dprint
     if debug:
         dprint(f"Dropping indices for {pipe}...")
-    if not pipe.columns:
-        warn(f"Unable to drop indices for {pipe} without columns.", stack=False)
+
+    if not pipe.indices:
+        warn(f"No indices to drop for {pipe}.", stack=False)
         return False
+
+    cols_to_include = set((columns or []) + (indices or [])) or None
+
     ix_queries = {
-        ix: queries
-        for ix, queries in self.get_drop_index_queries(pipe, debug=debug).items()
-        if indices is None or ix in indices
+        col: queries
+        for col, queries in self.get_drop_index_queries(pipe, debug=debug).items()
+        if cols_to_include is None or col in cols_to_include
     }
     success = True
-    for ix, queries in ix_queries.items():
-        ix_success = all(self.exec_queries(queries, debug=debug, silent=True))
+    for col, queries in ix_queries.items():
+        ix_success = all(self.exec_queries(queries, debug=debug, silent=(not debug)))
         if not ix_success:
             success = False
             if debug:
-                dprint(f"Failed to drop index on column: {ix}")
+                dprint(f"Failed to drop index on column: {col}")
     return success
 
 
+def get_pipe_index_names(self, pipe: mrsm.Pipe) -> Dict[str, str]:
+    """
+    Return a dictionary mapping index keys to their names on the database.
+
+    Returns
+    -------
+    A dictionary of index keys to column names.
+    """
+    from meerschaum.utils.sql import DEFAULT_SCHEMA_FLAVORS
+    _parameters = pipe.parameters
+    _index_template = _parameters.get('index_template', "IX_{schema_str}{target}_{column_names}")
+    _schema = self.get_pipe_schema(pipe)
+    if _schema is None:
+        _schema = (
+            DEFAULT_SCHEMA_FLAVORS.get(self.flavor, None)
+            if self.flavor != 'mssql'
+            else None
+        )
+    schema_str = '' if _schema is None else f'{_schema}_'
+    schema_str = ''
+    _indices = pipe.indices
+    _target = pipe.target
+    _column_names = {
+        ix: (
+            '_'.join(cols)
+            if isinstance(cols, (list, tuple))
+            else str(cols)
+        )
+        for ix, cols in _indices.items()
+        if cols
+    }
+    _index_names = {
+        ix: _index_template.format(
+            target=_target,
+            column_names=column_names,
+            connector_keys=pipe.connector_keys,
+            metric_key=pipe.metric_key,
+            location_key=pipe.location_key,
+            schema_str=schema_str,
+        )
+        for ix, column_names in _column_names.items()
+    }
+    ### NOTE: Skip any duplicate indices.
+    seen_index_names = {}
+    for ix, index_name in _index_names.items():
+        if index_name in seen_index_names:
+            continue
+        seen_index_names[index_name] = ix
+    return {
+        ix: index_name
+        for index_name, ix in seen_index_names.items()
+    }
+
 def get_create_index_queries(
     self,
     pipe: mrsm.Pipe,
@@ -407,7 +508,11 @@ def get_create_index_queries(
 
     upsert = pipe.parameters.get('upsert', False) and (self.flavor + '-upsert') in UPDATE_QUERIES
     static = pipe.parameters.get('static', False)
+    null_indices = pipe.parameters.get('null_indices', True)
     index_names = pipe.get_indices()
+    unique_index_name_unquoted = index_names.get('unique', None) or f'IX_{pipe.target}_unique'
+    if upsert:
+        _ = index_names.pop('unique', None)
     indices = pipe.indices
     existing_cols_types = pipe.get_columns_types(debug=debug)
     existing_cols_pd_types = {
@@ -420,11 +525,11 @@ def get_create_index_queries(
     existing_clustered_primary_keys = []
     for col, col_indices in existing_cols_indices.items():
         for col_ix_doc in col_indices:
-            existing_ix_names.add(col_ix_doc.get('name', None))
+            existing_ix_names.add(col_ix_doc.get('name', '').lower())
             if col_ix_doc.get('type', None) == 'PRIMARY KEY':
-                existing_primary_keys.append(col)
+                existing_primary_keys.append(col.lower())
                 if col_ix_doc.get('clustered', True):
-                    existing_clustered_primary_keys.append(col)
+                    existing_clustered_primary_keys.append(col.lower())
 
     _datetime = pipe.get_columns('datetime', error=False)
     _datetime_name = (
@@ -456,7 +561,7 @@ def get_create_index_queries(
         )
     )
     primary_key_db_type = (
-        get_db_type_from_pd_type(pipe.dtypes.get(primary_key, 'int'), self.flavor)
+        get_db_type_from_pd_type(pipe.dtypes.get(primary_key, 'int') or 'int', self.flavor)
         if primary_key
         else None
     )
@@ -471,6 +576,19 @@ def get_create_index_queries(
         if not existing_clustered_primary_keys and _datetime is not None
         else "NONCLUSTERED"
     )
+    include_columns_str = "\n    ,".join(
+        [
+            sql_item_name(col, flavor=self.flavor) for col in existing_cols_types
+            if col != _datetime
+        ]
+    ).rstrip(',')
+    include_clause = (
+        (
+            f"\nINCLUDE (\n    {include_columns_str}\n)"
+        )
+        if datetime_clustered == 'NONCLUSTERED'
+        else ''
+    )
 
     _id_index_name = (
         sql_item_name(index_names['id'], self.flavor, None)
@@ -516,7 +634,7 @@ def get_create_index_queries(
             if self.flavor == 'mssql':
                 dt_query = (
                     f"CREATE {datetime_clustered} INDEX {_datetime_index_name} "
-                    f"ON {_pipe_name} ({_datetime_name})"
+                    f"\nON {_pipe_name} ({_datetime_name}){include_clause}"
                 )
             else:
                 dt_query = (
@@ -530,7 +648,7 @@ def get_create_index_queries(
     primary_queries = []
     if (
         primary_key is not None
-        and primary_key not in existing_primary_keys
+        and primary_key.lower() not in existing_primary_keys
         and not static
     ):
         if autoincrement and primary_key not in existing_cols_pd_types:
@@ -659,7 +777,10 @@ def get_create_index_queries(
     other_index_names = {
         ix_key: ix_unquoted
         for ix_key, ix_unquoted in index_names.items()
-        if ix_key not in ('datetime', 'id', 'primary') and ix_unquoted not in existing_ix_names
+        if (
+            ix_key not in ('datetime', 'id', 'primary')
+            and ix_unquoted.lower() not in existing_ix_names
+        )
     }
     for ix_key, ix_unquoted in other_index_names.items():
         ix_name = sql_item_name(ix_unquoted, self.flavor, None)
@@ -684,24 +805,29 @@ def get_create_index_queries(
     coalesce_indices_cols_str = ', '.join(
         [
             (
-                "COALESCE("
-                + sql_item_name(ix, self.flavor)
-                + ", "
-                + get_null_replacement(existing_cols_types[ix], self.flavor)
-                + ") "
-            ) if ix_key != 'datetime' else (sql_item_name(ix, self.flavor))
+                (
+                    "COALESCE("
+                    + sql_item_name(ix, self.flavor)
+                    + ", "
+                    + get_null_replacement(existing_cols_types[ix], self.flavor)
+                    + ") "
+                )
+                if ix_key != 'datetime' and null_indices
+                else sql_item_name(ix, self.flavor)
+            )
             for ix_key, ix in pipe.columns.items()
             if ix and ix in existing_cols_types
         ]
     )
-    unique_index_name = sql_item_name(pipe.target + '_unique_index', self.flavor)
-    constraint_name = sql_item_name(pipe.target + '_constraint', self.flavor)
+    unique_index_name = sql_item_name(unique_index_name_unquoted, self.flavor)
+    constraint_name_unquoted = unique_index_name_unquoted.replace('IX_', 'UQ_')
+    constraint_name = sql_item_name(constraint_name_unquoted, self.flavor)
     add_constraint_query = (
         f"ALTER TABLE {_pipe_name} ADD CONSTRAINT {constraint_name} UNIQUE ({indices_cols_str})"
     )
     unique_index_cols_str = (
         indices_cols_str
-        if self.flavor not in COALESCE_UNIQUE_INDEX_FLAVORS
+        if self.flavor not in COALESCE_UNIQUE_INDEX_FLAVORS or not null_indices
         else coalesce_indices_cols_str
     )
     create_unique_index_query = (
@@ -737,21 +863,33 @@ def get_drop_index_queries(
         return {}
     if not pipe.exists(debug=debug):
         return {}
+
+    from collections import defaultdict
     from meerschaum.utils.sql import (
         sql_item_name,
         table_exists,
         hypertable_queries,
-        DROP_IF_EXISTS_FLAVORS,
+        DROP_INDEX_IF_EXISTS_FLAVORS,
     )
-    drop_queries = {}
+    drop_queries = defaultdict(lambda: [])
     schema = self.get_pipe_schema(pipe)
-    schema_prefix = (schema + '_') if schema else ''
+    index_schema = schema if self.flavor != 'mssql' else None
     indices = {
-        col: schema_prefix + ix
-        for col, ix in pipe.get_indices().items()
+        ix_key: ix
+        for ix_key, ix in pipe.get_indices().items()
     }
-    pipe_name = sql_item_name(pipe.target, self.flavor, self.get_pipe_schema(pipe))
+    cols_indices = pipe.get_columns_indices(debug=debug)
+    existing_indices = set()
+    clustered_ix = None
+    for col, ix_metas in cols_indices.items():
+        for ix_meta in ix_metas:
+            ix_name = ix_meta.get('name', None)
+            if ix_meta.get('clustered', False):
+                clustered_ix = ix_name
+            existing_indices.add(ix_name.lower())
+    pipe_name = sql_item_name(pipe.target, self.flavor, schema)
     pipe_name_no_schema = sql_item_name(pipe.target, self.flavor, None)
+    upsert = pipe.upsert
 
     if self.flavor not in hypertable_queries:
         is_hypertable = False
@@ -759,7 +897,7 @@ def get_drop_index_queries(
         is_hypertable_query = hypertable_queries[self.flavor].format(table_name=pipe_name)
         is_hypertable = self.value(is_hypertable_query, silent=True, debug=debug) is not None
 
-    if_exists_str = "IF EXISTS" if self.flavor in DROP_IF_EXISTS_FLAVORS else ""
+    if_exists_str = "IF EXISTS " if self.flavor in DROP_INDEX_IF_EXISTS_FLAVORS else ""
     if is_hypertable:
         nuke_queries = []
         temp_table = '_' + pipe.target + '_temp_migration'
@@ -769,21 +907,51 @@ def get_drop_index_queries(
             nuke_queries.append(f"DROP TABLE {if_exists_str} {temp_table_name}")
         nuke_queries += [
             f"SELECT * INTO {temp_table_name} FROM {pipe_name}",
-            f"DROP TABLE {if_exists_str} {pipe_name}",
+            f"DROP TABLE {if_exists_str}{pipe_name}",
             f"ALTER TABLE {temp_table_name} RENAME TO {pipe_name_no_schema}",
         ]
         nuke_ix_keys = ('datetime', 'id')
         nuked = False
         for ix_key in nuke_ix_keys:
             if ix_key in indices and not nuked:
-                drop_queries[ix_key] = nuke_queries
+                drop_queries[ix_key].extend(nuke_queries)
                 nuked = True
 
-    drop_queries.update({
-        ix_key: ["DROP INDEX " + sql_item_name(ix_unquoted, self.flavor, None)]
-        for ix_key, ix_unquoted in indices.items()
-        if ix_key not in drop_queries
-    })
+    for ix_key, ix_unquoted in indices.items():
+        if ix_key in drop_queries:
+            continue
+        if ix_unquoted.lower() not in existing_indices:
+            continue
+
+        if ix_key == 'unique' and upsert and self.flavor not in ('sqlite',) and not is_hypertable:
+            constraint_name_unquoted = ix_unquoted.replace('IX_', 'UQ_')
+            constraint_name = sql_item_name(constraint_name_unquoted, self.flavor)
+            constraint_or_index = (
+                "CONSTRAINT"
+                if self.flavor not in ('mysql', 'mariadb')
+                else 'INDEX'
+            )
+            drop_queries[ix_key].append(
+                f"ALTER TABLE {pipe_name}\n"
+                f"DROP {constraint_or_index} {constraint_name}"
+            )
+
+        query = (
+            (
+                f"ALTER TABLE {pipe_name}\n"
+                if self.flavor in ('mysql', 'mariadb')
+                else ''
+            )
+            + f"DROP INDEX {if_exists_str}"
+            + sql_item_name(ix_unquoted, self.flavor, index_schema)
+        )
+        if self.flavor == 'mssql':
+            query += f"\nON {pipe_name}"
+            if ix_unquoted == clustered_ix:
+                query += "\nWITH (ONLINE = ON, MAXDOP = 4)"
+        drop_queries[ix_key].append(query)
+
+
     return drop_queries
 
 
@@ -796,7 +964,7 @@ def delete_pipe(
     Delete a Pipe's registration.
     """
     from meerschaum.utils.packages import attempt_import
-    sqlalchemy = attempt_import('sqlalchemy')
+    sqlalchemy = attempt_import('sqlalchemy', lazy=False)
 
     if not pipe.id:
         return False, f"{pipe} is not registered."
@@ -1368,7 +1536,7 @@ def create_pipe_table_from_df(
     from meerschaum.utils.dtypes.sql import get_db_type_from_pd_type
     primary_key = pipe.columns.get('primary', None)
     primary_key_typ = (
-        pipe.dtypes.get(primary_key, str(df.dtypes.get(primary_key)))
+        pipe.dtypes.get(primary_key, str(df.dtypes.get(primary_key, 'int')))
         if primary_key
         else None
     )
@@ -1777,6 +1945,7 @@ def sync_pipe(
             patch_schema=self.internal_schema,
             datetime_col=(dt_col if dt_col in update_df.columns else None),
             identity_insert=(autoincrement and primary_key in update_df.columns),
+            null_indices=pipe.null_indices,
             debug=debug,
         )
         update_results = self.exec_queries(
@@ -2077,13 +2246,19 @@ def sync_pipe_inplace(
         _ = clean_up_temp_tables()
         return True, f"Inserted {new_count}, updated 0 rows."
 
-    dt_col_name_da = dateadd_str(flavor=self.flavor, begin=dt_col_name, db_type=dt_db_type)
+    min_dt_col_name_da = dateadd_str(
+        flavor=self.flavor, begin=f"MIN({dt_col_name})", db_type=dt_db_type,
+    )
+    max_dt_col_name_da = dateadd_str(
+        flavor=self.flavor, begin=f"MAX({dt_col_name})", db_type=dt_db_type,
+    )
+
     (new_dt_bounds_success, new_dt_bounds_msg), new_dt_bounds_results = session_execute(
         session,
         [
             "SELECT\n"
-            f"    MIN({dt_col_name_da}) AS {sql_item_name('min_dt', self.flavor)},\n"
-            f"    MAX({dt_col_name_da}) AS {sql_item_name('max_dt', self.flavor)}\n"
+            f"    {min_dt_col_name_da} AS {sql_item_name('min_dt', self.flavor)},\n"
+            f"    {max_dt_col_name_da} AS {sql_item_name('max_dt', self.flavor)}\n"
             f"FROM {temp_table_names['new' if not upsert else 'update']}\n"
             f"WHERE {dt_col_name} IS NOT NULL"
         ],
@@ -2350,6 +2525,7 @@ def sync_pipe_inplace(
             patch_schema=internal_schema,
             datetime_col=pipe.columns.get('datetime', None),
             flavor=self.flavor,
+            null_indices=pipe.null_indices,
             debug=debug,
         )
         if on_cols else []
@@ -2643,7 +2819,7 @@ def get_pipe_rowcount(
         _cols_names = ['*']
 
     src = (
-        f"SELECT {', '.join(_cols_names)} FROM {_pipe_name}"
+        f"SELECT {', '.join(_cols_names)}\nFROM {_pipe_name}"
         if not remote
         else get_pipe_query(pipe)
     )
@@ -2652,15 +2828,17 @@ def get_pipe_rowcount(
     if begin is not None or end is not None:
         query += "\nWHERE"
     if begin is not None:
-        query += f"""
-        {dt_name} >= {dateadd_str(self.flavor, datepart='minute', number=0, begin=begin, db_type=dt_db_type)}
-        """
+        query += (
+            f"\n    {dt_name} >= "
+            + dateadd_str(self.flavor, datepart='minute', number=0, begin=begin, db_type=dt_db_type)
+        )
     if end is not None and begin is not None:
-        query += "AND"
+        query += "\n    AND"
     if end is not None:
-        query += f"""
-        {dt_name} <  {dateadd_str(self.flavor, datepart='minute', number=0, begin=end, db_type=dt_db_type)}
-        """
+        query += (
+            f"\n    {dt_name} <  "
+            + dateadd_str(self.flavor, datepart='minute', number=0, begin=end, db_type=dt_db_type)
+        )
     if params is not None:
         from meerschaum.utils.sql import build_where
         existing_cols = pipe.get_columns_types(debug=debug)
@@ -2782,13 +2960,21 @@ def clear_pipe(
         valid_params = {k: v for k, v in params.items() if k in existing_cols}
     clear_query = (
         f"DELETE FROM {pipe_name}\nWHERE 1 = 1\n"
-        + ('  AND ' + build_where(valid_params, self, with_where=False) if valid_params else '')
+        + ('\n    AND ' + build_where(valid_params, self, with_where=False) if valid_params else '')
         + (
-            f'  AND {dt_name} >= ' + dateadd_str(self.flavor, 'day', 0, begin, db_type=dt_db_type)
-            if begin is not None else ''
+            (
+                f'\n    AND {dt_name} >= '
+                + dateadd_str(self.flavor, 'day', 0, begin, db_type=dt_db_type)
+            )
+            if begin is not None
+            else ''
         ) + (
-            f'  AND {dt_name} < ' + dateadd_str(self.flavor, 'day', 0, end, db_type=dt_db_type)
-            if end is not None else ''
+            (
+                f'\n    AND {dt_name} <  '
+                + dateadd_str(self.flavor, 'day', 0, end, db_type=dt_db_type)
+            )
+            if end is not None
+            else ''
         )
     )
     success = self.exec(clear_query, silent=True, debug=debug) is not None
@@ -2999,7 +3185,9 @@ def get_add_columns_queries(
         col: get_db_type_from_pd_type(
             df_cols_types[col],
             self.flavor
-        ) for col in new_cols
+        )
+        for col in new_cols
+        if col and df_cols_types.get(col, None)
     }
 
     alter_table_query = "ALTER TABLE " + sql_item_name(
@@ -3391,6 +3579,7 @@ def get_to_sql_dtype(
     return {
         col: get_db_type_from_pd_type(typ, self.flavor, as_sqlalchemy=True)
         for col, typ in df_dtypes.items()
+        if col and typ
     }