meerschaum 2.7.6__py3-none-any.whl → 2.7.8__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. meerschaum/actions/copy.py +1 -0
  2. meerschaum/actions/drop.py +100 -22
  3. meerschaum/actions/index.py +71 -0
  4. meerschaum/actions/register.py +8 -12
  5. meerschaum/actions/sql.py +1 -1
  6. meerschaum/api/routes/_pipes.py +18 -0
  7. meerschaum/api/routes/_plugins.py +1 -1
  8. meerschaum/api/routes/_users.py +62 -61
  9. meerschaum/config/_version.py +1 -1
  10. meerschaum/connectors/api/_pipes.py +20 -0
  11. meerschaum/connectors/sql/_SQLConnector.py +8 -12
  12. meerschaum/connectors/sql/_create_engine.py +1 -1
  13. meerschaum/connectors/sql/_fetch.py +9 -39
  14. meerschaum/connectors/sql/_instance.py +3 -3
  15. meerschaum/connectors/sql/_pipes.py +262 -70
  16. meerschaum/connectors/sql/_plugins.py +11 -16
  17. meerschaum/connectors/sql/_sql.py +60 -39
  18. meerschaum/connectors/sql/_uri.py +9 -9
  19. meerschaum/connectors/sql/_users.py +10 -12
  20. meerschaum/connectors/sql/tables/__init__.py +13 -14
  21. meerschaum/connectors/valkey/_ValkeyConnector.py +2 -2
  22. meerschaum/core/Pipe/__init__.py +12 -2
  23. meerschaum/core/Pipe/_attributes.py +32 -38
  24. meerschaum/core/Pipe/_drop.py +73 -2
  25. meerschaum/core/Pipe/_fetch.py +4 -0
  26. meerschaum/core/Pipe/_index.py +68 -0
  27. meerschaum/core/Pipe/_sync.py +16 -9
  28. meerschaum/utils/daemon/Daemon.py +9 -2
  29. meerschaum/utils/daemon/RotatingFile.py +3 -3
  30. meerschaum/utils/dataframe.py +42 -12
  31. meerschaum/utils/dtypes/__init__.py +144 -24
  32. meerschaum/utils/dtypes/sql.py +52 -9
  33. meerschaum/utils/formatting/__init__.py +2 -2
  34. meerschaum/utils/formatting/_pprint.py +12 -11
  35. meerschaum/utils/misc.py +16 -18
  36. meerschaum/utils/prompt.py +1 -1
  37. meerschaum/utils/sql.py +106 -42
  38. {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/METADATA +14 -2
  39. {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/RECORD +45 -43
  40. {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/WHEEL +1 -1
  41. {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/LICENSE +0 -0
  42. {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/NOTICE +0 -0
  43. {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/entry_points.txt +0 -0
  44. {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/top_level.txt +0 -0
  45. {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/zip-safe +0 -0
@@ -55,7 +55,7 @@ def register_pipe(
55
55
  parameters = {}
56
56
 
57
57
  import json
58
- sqlalchemy = attempt_import('sqlalchemy')
58
+ sqlalchemy = attempt_import('sqlalchemy', lazy=False)
59
59
  values = {
60
60
  'connector_keys' : pipe.connector_keys,
61
61
  'metric_key' : pipe.metric_key,
@@ -118,7 +118,7 @@ def edit_pipe(
118
118
  pipes_tbl = get_tables(mrsm_instance=self, create=(not pipe.temporary), debug=debug)['pipes']
119
119
 
120
120
  import json
121
- sqlalchemy = attempt_import('sqlalchemy')
121
+ sqlalchemy = attempt_import('sqlalchemy', lazy=False)
122
122
 
123
123
  values = {
124
124
  'parameters': (
@@ -176,7 +176,10 @@ def fetch_pipes_keys(
176
176
  from meerschaum.config.static import STATIC_CONFIG
177
177
  import json
178
178
  from copy import deepcopy
179
- sqlalchemy, sqlalchemy_sql_functions = attempt_import('sqlalchemy', 'sqlalchemy.sql.functions')
179
+ sqlalchemy, sqlalchemy_sql_functions = attempt_import(
180
+ 'sqlalchemy',
181
+ 'sqlalchemy.sql.functions', lazy=False,
182
+ )
180
183
  coalesce = sqlalchemy_sql_functions.coalesce
181
184
 
182
185
  if connector_keys is None:
@@ -246,7 +249,7 @@ def fetch_pipes_keys(
246
249
 
247
250
  q = sqlalchemy.select(*select_cols).where(sqlalchemy.and_(True, *_where))
248
251
  for c, vals in cols.items():
249
- if not isinstance(vals, (list, tuple)) or not vals or not c in pipes_tbl.c:
252
+ if not isinstance(vals, (list, tuple)) or not vals or c not in pipes_tbl.c:
250
253
  continue
251
254
  _in_vals, _ex_vals = separate_negation_values(vals)
252
255
  q = q.where(coalesce(pipes_tbl.c[c], 'None').in_(_in_vals)) if _in_vals else q
@@ -306,9 +309,28 @@ def fetch_pipes_keys(
306
309
  return [(row[0], row[1], row[2]) for row in rows]
307
310
 
308
311
 
312
+ def create_pipe_indices(
313
+ self,
314
+ pipe: mrsm.Pipe,
315
+ columns: Optional[List[str]] = None,
316
+ debug: bool = False,
317
+ ) -> SuccessTuple:
318
+ """
319
+ Create a pipe's indices.
320
+ """
321
+ success = self.create_indices(pipe, columns=columns, debug=debug)
322
+ msg = (
323
+ "Success"
324
+ if success
325
+ else f"Failed to create indices for {pipe}."
326
+ )
327
+ return success, msg
328
+
329
+
309
330
  def create_indices(
310
331
  self,
311
332
  pipe: mrsm.Pipe,
333
+ columns: Optional[List[str]] = None,
312
334
  indices: Optional[List[str]] = None,
313
335
  debug: bool = False
314
336
  ) -> bool:
@@ -318,29 +340,51 @@ def create_indices(
318
340
  from meerschaum.utils.debug import dprint
319
341
  if debug:
320
342
  dprint(f"Creating indices for {pipe}...")
343
+
321
344
  if not pipe.indices:
322
345
  warn(f"{pipe} has no index columns; skipping index creation.", stack=False)
323
346
  return True
324
347
 
348
+ cols_to_include = set((columns or []) + (indices or [])) or None
349
+
325
350
  _ = pipe.__dict__.pop('_columns_indices', None)
326
351
  ix_queries = {
327
- ix: queries
328
- for ix, queries in self.get_create_index_queries(pipe, debug=debug).items()
329
- if indices is None or ix in indices
352
+ col: queries
353
+ for col, queries in self.get_create_index_queries(pipe, debug=debug).items()
354
+ if cols_to_include is None or col in cols_to_include
330
355
  }
331
356
  success = True
332
- for ix, queries in ix_queries.items():
357
+ for col, queries in ix_queries.items():
333
358
  ix_success = all(self.exec_queries(queries, debug=debug, silent=False))
334
359
  success = success and ix_success
335
360
  if not ix_success:
336
- warn(f"Failed to create index on column: {ix}")
361
+ warn(f"Failed to create index on column: {col}")
337
362
 
338
363
  return success
339
364
 
340
365
 
366
+ def drop_pipe_indices(
367
+ self,
368
+ pipe: mrsm.Pipe,
369
+ columns: Optional[List[str]] = None,
370
+ debug: bool = False,
371
+ ) -> SuccessTuple:
372
+ """
373
+ Drop a pipe's indices.
374
+ """
375
+ success = self.drop_indices(pipe, columns=columns, debug=debug)
376
+ msg = (
377
+ "Success"
378
+ if success
379
+ else f"Failed to drop indices for {pipe}."
380
+ )
381
+ return success, msg
382
+
383
+
341
384
  def drop_indices(
342
385
  self,
343
386
  pipe: mrsm.Pipe,
387
+ columns: Optional[List[str]] = None,
344
388
  indices: Optional[List[str]] = None,
345
389
  debug: bool = False
346
390
  ) -> bool:
@@ -350,24 +394,81 @@ def drop_indices(
350
394
  from meerschaum.utils.debug import dprint
351
395
  if debug:
352
396
  dprint(f"Dropping indices for {pipe}...")
353
- if not pipe.columns:
354
- warn(f"Unable to drop indices for {pipe} without columns.", stack=False)
397
+
398
+ if not pipe.indices:
399
+ warn(f"No indices to drop for {pipe}.", stack=False)
355
400
  return False
401
+
402
+ cols_to_include = set((columns or []) + (indices or [])) or None
403
+
356
404
  ix_queries = {
357
- ix: queries
358
- for ix, queries in self.get_drop_index_queries(pipe, debug=debug).items()
359
- if indices is None or ix in indices
405
+ col: queries
406
+ for col, queries in self.get_drop_index_queries(pipe, debug=debug).items()
407
+ if cols_to_include is None or col in cols_to_include
360
408
  }
361
409
  success = True
362
- for ix, queries in ix_queries.items():
363
- ix_success = all(self.exec_queries(queries, debug=debug, silent=True))
410
+ for col, queries in ix_queries.items():
411
+ ix_success = all(self.exec_queries(queries, debug=debug, silent=(not debug)))
364
412
  if not ix_success:
365
413
  success = False
366
414
  if debug:
367
- dprint(f"Failed to drop index on column: {ix}")
415
+ dprint(f"Failed to drop index on column: {col}")
368
416
  return success
369
417
 
370
418
 
419
+ def get_pipe_index_names(self, pipe: mrsm.Pipe) -> Dict[str, str]:
420
+ """
421
+ Return a dictionary mapping index keys to their names on the database.
422
+
423
+ Returns
424
+ -------
425
+ A dictionary of index keys to column names.
426
+ """
427
+ from meerschaum.utils.sql import DEFAULT_SCHEMA_FLAVORS
428
+ _parameters = pipe.parameters
429
+ _index_template = _parameters.get('index_template', "IX_{schema_str}{target}_{column_names}")
430
+ _schema = self.get_pipe_schema(pipe)
431
+ if _schema is None:
432
+ _schema = (
433
+ DEFAULT_SCHEMA_FLAVORS.get(self.flavor, None)
434
+ if self.flavor != 'mssql'
435
+ else None
436
+ )
437
+ schema_str = '' if _schema is None else f'{_schema}_'
438
+ schema_str = ''
439
+ _indices = pipe.indices
440
+ _target = pipe.target
441
+ _column_names = {
442
+ ix: (
443
+ '_'.join(cols)
444
+ if isinstance(cols, (list, tuple))
445
+ else str(cols)
446
+ )
447
+ for ix, cols in _indices.items()
448
+ if cols
449
+ }
450
+ _index_names = {
451
+ ix: _index_template.format(
452
+ target=_target,
453
+ column_names=column_names,
454
+ connector_keys=pipe.connector_keys,
455
+ metric_key=pipe.metric_key,
456
+ location_key=pipe.location_key,
457
+ schema_str=schema_str,
458
+ )
459
+ for ix, column_names in _column_names.items()
460
+ }
461
+ ### NOTE: Skip any duplicate indices.
462
+ seen_index_names = {}
463
+ for ix, index_name in _index_names.items():
464
+ if index_name in seen_index_names:
465
+ continue
466
+ seen_index_names[index_name] = ix
467
+ return {
468
+ ix: index_name
469
+ for index_name, ix in seen_index_names.items()
470
+ }
471
+
371
472
  def get_create_index_queries(
372
473
  self,
373
474
  pipe: mrsm.Pipe,
@@ -407,7 +508,11 @@ def get_create_index_queries(
407
508
 
408
509
  upsert = pipe.parameters.get('upsert', False) and (self.flavor + '-upsert') in UPDATE_QUERIES
409
510
  static = pipe.parameters.get('static', False)
511
+ null_indices = pipe.parameters.get('null_indices', True)
410
512
  index_names = pipe.get_indices()
513
+ unique_index_name_unquoted = index_names.get('unique', None) or f'IX_{pipe.target}_unique'
514
+ if upsert:
515
+ _ = index_names.pop('unique', None)
411
516
  indices = pipe.indices
412
517
  existing_cols_types = pipe.get_columns_types(debug=debug)
413
518
  existing_cols_pd_types = {
@@ -420,11 +525,11 @@ def get_create_index_queries(
420
525
  existing_clustered_primary_keys = []
421
526
  for col, col_indices in existing_cols_indices.items():
422
527
  for col_ix_doc in col_indices:
423
- existing_ix_names.add(col_ix_doc.get('name', None))
528
+ existing_ix_names.add(col_ix_doc.get('name', '').lower())
424
529
  if col_ix_doc.get('type', None) == 'PRIMARY KEY':
425
- existing_primary_keys.append(col)
530
+ existing_primary_keys.append(col.lower())
426
531
  if col_ix_doc.get('clustered', True):
427
- existing_clustered_primary_keys.append(col)
532
+ existing_clustered_primary_keys.append(col.lower())
428
533
 
429
534
  _datetime = pipe.get_columns('datetime', error=False)
430
535
  _datetime_name = (
@@ -456,7 +561,7 @@ def get_create_index_queries(
456
561
  )
457
562
  )
458
563
  primary_key_db_type = (
459
- get_db_type_from_pd_type(pipe.dtypes.get(primary_key, 'int'), self.flavor)
564
+ get_db_type_from_pd_type(pipe.dtypes.get(primary_key, 'int') or 'int', self.flavor)
460
565
  if primary_key
461
566
  else None
462
567
  )
@@ -471,6 +576,19 @@ def get_create_index_queries(
471
576
  if not existing_clustered_primary_keys and _datetime is not None
472
577
  else "NONCLUSTERED"
473
578
  )
579
+ include_columns_str = "\n ,".join(
580
+ [
581
+ sql_item_name(col, flavor=self.flavor) for col in existing_cols_types
582
+ if col != _datetime
583
+ ]
584
+ ).rstrip(',')
585
+ include_clause = (
586
+ (
587
+ f"\nINCLUDE (\n {include_columns_str}\n)"
588
+ )
589
+ if datetime_clustered == 'NONCLUSTERED'
590
+ else ''
591
+ )
474
592
 
475
593
  _id_index_name = (
476
594
  sql_item_name(index_names['id'], self.flavor, None)
@@ -516,7 +634,7 @@ def get_create_index_queries(
516
634
  if self.flavor == 'mssql':
517
635
  dt_query = (
518
636
  f"CREATE {datetime_clustered} INDEX {_datetime_index_name} "
519
- f"ON {_pipe_name} ({_datetime_name})"
637
+ f"\nON {_pipe_name} ({_datetime_name}){include_clause}"
520
638
  )
521
639
  else:
522
640
  dt_query = (
@@ -530,7 +648,7 @@ def get_create_index_queries(
530
648
  primary_queries = []
531
649
  if (
532
650
  primary_key is not None
533
- and primary_key not in existing_primary_keys
651
+ and primary_key.lower() not in existing_primary_keys
534
652
  and not static
535
653
  ):
536
654
  if autoincrement and primary_key not in existing_cols_pd_types:
@@ -659,7 +777,10 @@ def get_create_index_queries(
659
777
  other_index_names = {
660
778
  ix_key: ix_unquoted
661
779
  for ix_key, ix_unquoted in index_names.items()
662
- if ix_key not in ('datetime', 'id', 'primary') and ix_unquoted not in existing_ix_names
780
+ if (
781
+ ix_key not in ('datetime', 'id', 'primary')
782
+ and ix_unquoted.lower() not in existing_ix_names
783
+ )
663
784
  }
664
785
  for ix_key, ix_unquoted in other_index_names.items():
665
786
  ix_name = sql_item_name(ix_unquoted, self.flavor, None)
@@ -684,24 +805,29 @@ def get_create_index_queries(
684
805
  coalesce_indices_cols_str = ', '.join(
685
806
  [
686
807
  (
687
- "COALESCE("
688
- + sql_item_name(ix, self.flavor)
689
- + ", "
690
- + get_null_replacement(existing_cols_types[ix], self.flavor)
691
- + ") "
692
- ) if ix_key != 'datetime' else (sql_item_name(ix, self.flavor))
808
+ (
809
+ "COALESCE("
810
+ + sql_item_name(ix, self.flavor)
811
+ + ", "
812
+ + get_null_replacement(existing_cols_types[ix], self.flavor)
813
+ + ") "
814
+ )
815
+ if ix_key != 'datetime' and null_indices
816
+ else sql_item_name(ix, self.flavor)
817
+ )
693
818
  for ix_key, ix in pipe.columns.items()
694
819
  if ix and ix in existing_cols_types
695
820
  ]
696
821
  )
697
- unique_index_name = sql_item_name(pipe.target + '_unique_index', self.flavor)
698
- constraint_name = sql_item_name(pipe.target + '_constraint', self.flavor)
822
+ unique_index_name = sql_item_name(unique_index_name_unquoted, self.flavor)
823
+ constraint_name_unquoted = unique_index_name_unquoted.replace('IX_', 'UQ_')
824
+ constraint_name = sql_item_name(constraint_name_unquoted, self.flavor)
699
825
  add_constraint_query = (
700
826
  f"ALTER TABLE {_pipe_name} ADD CONSTRAINT {constraint_name} UNIQUE ({indices_cols_str})"
701
827
  )
702
828
  unique_index_cols_str = (
703
829
  indices_cols_str
704
- if self.flavor not in COALESCE_UNIQUE_INDEX_FLAVORS
830
+ if self.flavor not in COALESCE_UNIQUE_INDEX_FLAVORS or not null_indices
705
831
  else coalesce_indices_cols_str
706
832
  )
707
833
  create_unique_index_query = (
@@ -737,21 +863,33 @@ def get_drop_index_queries(
737
863
  return {}
738
864
  if not pipe.exists(debug=debug):
739
865
  return {}
866
+
867
+ from collections import defaultdict
740
868
  from meerschaum.utils.sql import (
741
869
  sql_item_name,
742
870
  table_exists,
743
871
  hypertable_queries,
744
- DROP_IF_EXISTS_FLAVORS,
872
+ DROP_INDEX_IF_EXISTS_FLAVORS,
745
873
  )
746
- drop_queries = {}
874
+ drop_queries = defaultdict(lambda: [])
747
875
  schema = self.get_pipe_schema(pipe)
748
- schema_prefix = (schema + '_') if schema else ''
876
+ index_schema = schema if self.flavor != 'mssql' else None
749
877
  indices = {
750
- col: schema_prefix + ix
751
- for col, ix in pipe.get_indices().items()
878
+ ix_key: ix
879
+ for ix_key, ix in pipe.get_indices().items()
752
880
  }
753
- pipe_name = sql_item_name(pipe.target, self.flavor, self.get_pipe_schema(pipe))
881
+ cols_indices = pipe.get_columns_indices(debug=debug)
882
+ existing_indices = set()
883
+ clustered_ix = None
884
+ for col, ix_metas in cols_indices.items():
885
+ for ix_meta in ix_metas:
886
+ ix_name = ix_meta.get('name', None)
887
+ if ix_meta.get('clustered', False):
888
+ clustered_ix = ix_name
889
+ existing_indices.add(ix_name.lower())
890
+ pipe_name = sql_item_name(pipe.target, self.flavor, schema)
754
891
  pipe_name_no_schema = sql_item_name(pipe.target, self.flavor, None)
892
+ upsert = pipe.upsert
755
893
 
756
894
  if self.flavor not in hypertable_queries:
757
895
  is_hypertable = False
@@ -759,7 +897,7 @@ def get_drop_index_queries(
759
897
  is_hypertable_query = hypertable_queries[self.flavor].format(table_name=pipe_name)
760
898
  is_hypertable = self.value(is_hypertable_query, silent=True, debug=debug) is not None
761
899
 
762
- if_exists_str = "IF EXISTS" if self.flavor in DROP_IF_EXISTS_FLAVORS else ""
900
+ if_exists_str = "IF EXISTS " if self.flavor in DROP_INDEX_IF_EXISTS_FLAVORS else ""
763
901
  if is_hypertable:
764
902
  nuke_queries = []
765
903
  temp_table = '_' + pipe.target + '_temp_migration'
@@ -769,21 +907,51 @@ def get_drop_index_queries(
769
907
  nuke_queries.append(f"DROP TABLE {if_exists_str} {temp_table_name}")
770
908
  nuke_queries += [
771
909
  f"SELECT * INTO {temp_table_name} FROM {pipe_name}",
772
- f"DROP TABLE {if_exists_str} {pipe_name}",
910
+ f"DROP TABLE {if_exists_str}{pipe_name}",
773
911
  f"ALTER TABLE {temp_table_name} RENAME TO {pipe_name_no_schema}",
774
912
  ]
775
913
  nuke_ix_keys = ('datetime', 'id')
776
914
  nuked = False
777
915
  for ix_key in nuke_ix_keys:
778
916
  if ix_key in indices and not nuked:
779
- drop_queries[ix_key] = nuke_queries
917
+ drop_queries[ix_key].extend(nuke_queries)
780
918
  nuked = True
781
919
 
782
- drop_queries.update({
783
- ix_key: ["DROP INDEX " + sql_item_name(ix_unquoted, self.flavor, None)]
784
- for ix_key, ix_unquoted in indices.items()
785
- if ix_key not in drop_queries
786
- })
920
+ for ix_key, ix_unquoted in indices.items():
921
+ if ix_key in drop_queries:
922
+ continue
923
+ if ix_unquoted.lower() not in existing_indices:
924
+ continue
925
+
926
+ if ix_key == 'unique' and upsert and self.flavor not in ('sqlite',) and not is_hypertable:
927
+ constraint_name_unquoted = ix_unquoted.replace('IX_', 'UQ_')
928
+ constraint_name = sql_item_name(constraint_name_unquoted, self.flavor)
929
+ constraint_or_index = (
930
+ "CONSTRAINT"
931
+ if self.flavor not in ('mysql', 'mariadb')
932
+ else 'INDEX'
933
+ )
934
+ drop_queries[ix_key].append(
935
+ f"ALTER TABLE {pipe_name}\n"
936
+ f"DROP {constraint_or_index} {constraint_name}"
937
+ )
938
+
939
+ query = (
940
+ (
941
+ f"ALTER TABLE {pipe_name}\n"
942
+ if self.flavor in ('mysql', 'mariadb')
943
+ else ''
944
+ )
945
+ + f"DROP INDEX {if_exists_str}"
946
+ + sql_item_name(ix_unquoted, self.flavor, index_schema)
947
+ )
948
+ if self.flavor == 'mssql':
949
+ query += f"\nON {pipe_name}"
950
+ if ix_unquoted == clustered_ix:
951
+ query += "\nWITH (ONLINE = ON, MAXDOP = 4)"
952
+ drop_queries[ix_key].append(query)
953
+
954
+
787
955
  return drop_queries
788
956
 
789
957
 
@@ -796,7 +964,7 @@ def delete_pipe(
796
964
  Delete a Pipe's registration.
797
965
  """
798
966
  from meerschaum.utils.packages import attempt_import
799
- sqlalchemy = attempt_import('sqlalchemy')
967
+ sqlalchemy = attempt_import('sqlalchemy', lazy=False)
800
968
 
801
969
  if not pipe.id:
802
970
  return False, f"{pipe} is not registered."
@@ -957,7 +1125,7 @@ def get_pipe_data(
957
1125
  numeric_columns = [
958
1126
  col
959
1127
  for col, typ in pipe.dtypes.items()
960
- if typ == 'numeric' and col in dtypes
1128
+ if typ.startswith('numeric') and col in dtypes
961
1129
  ]
962
1130
  uuid_columns = [
963
1131
  col
@@ -1368,7 +1536,7 @@ def create_pipe_table_from_df(
1368
1536
  from meerschaum.utils.dtypes.sql import get_db_type_from_pd_type
1369
1537
  primary_key = pipe.columns.get('primary', None)
1370
1538
  primary_key_typ = (
1371
- pipe.dtypes.get(primary_key, str(df.dtypes.get(primary_key)))
1539
+ pipe.dtypes.get(primary_key, str(df.dtypes.get(primary_key, 'int')))
1372
1540
  if primary_key
1373
1541
  else None
1374
1542
  )
@@ -1719,7 +1887,10 @@ def sync_pipe(
1719
1887
  warn(f"Could not reset auto-incrementing primary key for {pipe}.", stack=False)
1720
1888
 
1721
1889
  if update_df is not None and len(update_df) > 0:
1722
- temp_target = self.get_temporary_target(pipe.target, label='update')
1890
+ temp_target = self.get_temporary_target(
1891
+ pipe.target,
1892
+ label=('update' if not upsert else 'upsert'),
1893
+ )
1723
1894
  self._log_temporary_tables_creation(temp_target, create=(not pipe.temporary), debug=debug)
1724
1895
  temp_pipe = Pipe(
1725
1896
  pipe.connector_keys.replace(':', '_') + '_', pipe.metric_key, pipe.location_key,
@@ -1777,6 +1948,7 @@ def sync_pipe(
1777
1948
  patch_schema=self.internal_schema,
1778
1949
  datetime_col=(dt_col if dt_col in update_df.columns else None),
1779
1950
  identity_insert=(autoincrement and primary_key in update_df.columns),
1951
+ null_indices=pipe.null_indices,
1780
1952
  debug=debug,
1781
1953
  )
1782
1954
  update_results = self.exec_queries(
@@ -2077,13 +2249,19 @@ def sync_pipe_inplace(
2077
2249
  _ = clean_up_temp_tables()
2078
2250
  return True, f"Inserted {new_count}, updated 0 rows."
2079
2251
 
2080
- dt_col_name_da = dateadd_str(flavor=self.flavor, begin=dt_col_name, db_type=dt_db_type)
2252
+ min_dt_col_name_da = dateadd_str(
2253
+ flavor=self.flavor, begin=f"MIN({dt_col_name})", db_type=dt_db_type,
2254
+ )
2255
+ max_dt_col_name_da = dateadd_str(
2256
+ flavor=self.flavor, begin=f"MAX({dt_col_name})", db_type=dt_db_type,
2257
+ )
2258
+
2081
2259
  (new_dt_bounds_success, new_dt_bounds_msg), new_dt_bounds_results = session_execute(
2082
2260
  session,
2083
2261
  [
2084
2262
  "SELECT\n"
2085
- f" MIN({dt_col_name_da}) AS {sql_item_name('min_dt', self.flavor)},\n"
2086
- f" MAX({dt_col_name_da}) AS {sql_item_name('max_dt', self.flavor)}\n"
2263
+ f" {min_dt_col_name_da} AS {sql_item_name('min_dt', self.flavor)},\n"
2264
+ f" {max_dt_col_name_da} AS {sql_item_name('max_dt', self.flavor)}\n"
2087
2265
  f"FROM {temp_table_names['new' if not upsert else 'update']}\n"
2088
2266
  f"WHERE {dt_col_name} IS NOT NULL"
2089
2267
  ],
@@ -2350,6 +2528,7 @@ def sync_pipe_inplace(
2350
2528
  patch_schema=internal_schema,
2351
2529
  datetime_col=pipe.columns.get('datetime', None),
2352
2530
  flavor=self.flavor,
2531
+ null_indices=pipe.null_indices,
2353
2532
  debug=debug,
2354
2533
  )
2355
2534
  if on_cols else []
@@ -2643,7 +2822,7 @@ def get_pipe_rowcount(
2643
2822
  _cols_names = ['*']
2644
2823
 
2645
2824
  src = (
2646
- f"SELECT {', '.join(_cols_names)} FROM {_pipe_name}"
2825
+ f"SELECT {', '.join(_cols_names)}\nFROM {_pipe_name}"
2647
2826
  if not remote
2648
2827
  else get_pipe_query(pipe)
2649
2828
  )
@@ -2652,15 +2831,17 @@ def get_pipe_rowcount(
2652
2831
  if begin is not None or end is not None:
2653
2832
  query += "\nWHERE"
2654
2833
  if begin is not None:
2655
- query += f"""
2656
- {dt_name} >= {dateadd_str(self.flavor, datepart='minute', number=0, begin=begin, db_type=dt_db_type)}
2657
- """
2834
+ query += (
2835
+ f"\n {dt_name} >= "
2836
+ + dateadd_str(self.flavor, datepart='minute', number=0, begin=begin, db_type=dt_db_type)
2837
+ )
2658
2838
  if end is not None and begin is not None:
2659
- query += "AND"
2839
+ query += "\n AND"
2660
2840
  if end is not None:
2661
- query += f"""
2662
- {dt_name} < {dateadd_str(self.flavor, datepart='minute', number=0, begin=end, db_type=dt_db_type)}
2663
- """
2841
+ query += (
2842
+ f"\n {dt_name} < "
2843
+ + dateadd_str(self.flavor, datepart='minute', number=0, begin=end, db_type=dt_db_type)
2844
+ )
2664
2845
  if params is not None:
2665
2846
  from meerschaum.utils.sql import build_where
2666
2847
  existing_cols = pipe.get_columns_types(debug=debug)
@@ -2782,13 +2963,21 @@ def clear_pipe(
2782
2963
  valid_params = {k: v for k, v in params.items() if k in existing_cols}
2783
2964
  clear_query = (
2784
2965
  f"DELETE FROM {pipe_name}\nWHERE 1 = 1\n"
2785
- + (' AND ' + build_where(valid_params, self, with_where=False) if valid_params else '')
2966
+ + ('\n AND ' + build_where(valid_params, self, with_where=False) if valid_params else '')
2786
2967
  + (
2787
- f' AND {dt_name} >= ' + dateadd_str(self.flavor, 'day', 0, begin, db_type=dt_db_type)
2788
- if begin is not None else ''
2968
+ (
2969
+ f'\n AND {dt_name} >= '
2970
+ + dateadd_str(self.flavor, 'day', 0, begin, db_type=dt_db_type)
2971
+ )
2972
+ if begin is not None
2973
+ else ''
2789
2974
  ) + (
2790
- f' AND {dt_name} < ' + dateadd_str(self.flavor, 'day', 0, end, db_type=dt_db_type)
2791
- if end is not None else ''
2975
+ (
2976
+ f'\n AND {dt_name} < '
2977
+ + dateadd_str(self.flavor, 'day', 0, end, db_type=dt_db_type)
2978
+ )
2979
+ if end is not None
2980
+ else ''
2792
2981
  )
2793
2982
  )
2794
2983
  success = self.exec(clear_query, silent=True, debug=debug) is not None
@@ -2999,7 +3188,9 @@ def get_add_columns_queries(
2999
3188
  col: get_db_type_from_pd_type(
3000
3189
  df_cols_types[col],
3001
3190
  self.flavor
3002
- ) for col in new_cols
3191
+ )
3192
+ for col in new_cols
3193
+ if col and df_cols_types.get(col, None)
3003
3194
  }
3004
3195
 
3005
3196
  alter_table_query = "ALTER TABLE " + sql_item_name(
@@ -3086,7 +3277,7 @@ def get_alter_columns_queries(
3086
3277
  else [
3087
3278
  col
3088
3279
  for col, typ in df.items()
3089
- if typ == 'numeric'
3280
+ if typ.startswith('numeric')
3090
3281
  ]
3091
3282
  )
3092
3283
  df_cols_types = (
@@ -3166,7 +3357,7 @@ def get_alter_columns_queries(
3166
3357
  + f"{edit_msg}"
3167
3358
  )
3168
3359
  else:
3169
- numeric_cols.extend([col for col, typ in pipe.dtypes.items() if typ == 'numeric'])
3360
+ numeric_cols.extend([col for col, typ in pipe.dtypes.items() if typ.startswith('numeric')])
3170
3361
 
3171
3362
  numeric_type = get_db_type_from_pd_type('numeric', self.flavor, as_sqlalchemy=False)
3172
3363
  text_type = get_db_type_from_pd_type('str', self.flavor, as_sqlalchemy=False)
@@ -3391,6 +3582,7 @@ def get_to_sql_dtype(
3391
3582
  return {
3392
3583
  col: get_db_type_from_pd_type(typ, self.flavor, as_sqlalchemy=True)
3393
3584
  for col, typ in df_dtypes.items()
3585
+ if col and typ
3394
3586
  }
3395
3587
 
3396
3588