meerschaum 2.7.6__py3-none-any.whl → 2.7.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. meerschaum/actions/drop.py +100 -22
  2. meerschaum/actions/index.py +71 -0
  3. meerschaum/actions/register.py +8 -12
  4. meerschaum/actions/sql.py +1 -1
  5. meerschaum/api/routes/_pipes.py +18 -0
  6. meerschaum/api/routes/_plugins.py +1 -1
  7. meerschaum/api/routes/_users.py +62 -61
  8. meerschaum/config/_version.py +1 -1
  9. meerschaum/connectors/api/_pipes.py +20 -0
  10. meerschaum/connectors/sql/_SQLConnector.py +6 -3
  11. meerschaum/connectors/sql/_create_engine.py +1 -1
  12. meerschaum/connectors/sql/_fetch.py +4 -9
  13. meerschaum/connectors/sql/_instance.py +3 -3
  14. meerschaum/connectors/sql/_pipes.py +255 -66
  15. meerschaum/connectors/sql/_plugins.py +11 -16
  16. meerschaum/connectors/sql/_sql.py +4 -8
  17. meerschaum/connectors/sql/_uri.py +9 -9
  18. meerschaum/connectors/sql/_users.py +10 -12
  19. meerschaum/connectors/sql/tables/__init__.py +13 -14
  20. meerschaum/core/Pipe/__init__.py +12 -2
  21. meerschaum/core/Pipe/_attributes.py +32 -38
  22. meerschaum/core/Pipe/_drop.py +73 -2
  23. meerschaum/core/Pipe/_index.py +68 -0
  24. meerschaum/utils/dtypes/sql.py +2 -2
  25. meerschaum/utils/sql.py +80 -34
  26. {meerschaum-2.7.6.dist-info → meerschaum-2.7.7.dist-info}/METADATA +14 -2
  27. {meerschaum-2.7.6.dist-info → meerschaum-2.7.7.dist-info}/RECORD +33 -31
  28. {meerschaum-2.7.6.dist-info → meerschaum-2.7.7.dist-info}/WHEEL +1 -1
  29. {meerschaum-2.7.6.dist-info → meerschaum-2.7.7.dist-info}/LICENSE +0 -0
  30. {meerschaum-2.7.6.dist-info → meerschaum-2.7.7.dist-info}/NOTICE +0 -0
  31. {meerschaum-2.7.6.dist-info → meerschaum-2.7.7.dist-info}/entry_points.txt +0 -0
  32. {meerschaum-2.7.6.dist-info → meerschaum-2.7.7.dist-info}/top_level.txt +0 -0
  33. {meerschaum-2.7.6.dist-info → meerschaum-2.7.7.dist-info}/zip-safe +0 -0
@@ -148,7 +148,6 @@ def get_pipe_metadef(
148
148
  from meerschaum.utils.warnings import warn
149
149
  from meerschaum.utils.sql import sql_item_name, dateadd_str, build_where
150
150
  from meerschaum.utils.dtypes.sql import get_db_type_from_pd_type
151
- from meerschaum.utils.misc import is_int
152
151
  from meerschaum.config import get_config
153
152
 
154
153
  dt_col = pipe.columns.get('datetime', None)
@@ -191,7 +190,7 @@ def get_pipe_metadef(
191
190
  else begin
192
191
  )
193
192
 
194
- if begin and end and begin >= end:
193
+ if begin not in (None, '') and end is not None and begin >= end:
195
194
  begin = None
196
195
 
197
196
  if dt_name:
@@ -203,7 +202,7 @@ def get_pipe_metadef(
203
202
  begin=begin,
204
203
  db_type=db_dt_typ,
205
204
  )
206
- if begin
205
+ if begin not in ('', None)
207
206
  else None
208
207
  )
209
208
  end_da = (
@@ -214,7 +213,7 @@ def get_pipe_metadef(
214
213
  begin=end,
215
214
  db_type=db_dt_typ,
216
215
  )
217
- if end
216
+ if end is not None
218
217
  else None
219
218
  )
220
219
 
@@ -228,11 +227,7 @@ def get_pipe_metadef(
228
227
 
229
228
  has_where = 'where' in meta_def.lower()[meta_def.lower().rfind('definition'):]
230
229
  if dt_name and (begin_da or end_da):
231
- definition_dt_name = (
232
- dateadd_str(self.flavor, 'minute', 0, f"{definition_name}.{dt_name}", db_type=db_dt_typ)
233
- if not is_int((begin_da or end_da))
234
- else f"{definition_name}.{dt_name}"
235
- )
230
+ definition_dt_name = f"{definition_name}.{dt_name}"
236
231
  meta_def += "\n" + ("AND" if has_where else "WHERE") + " "
237
232
  has_where = True
238
233
  if begin_da:
@@ -25,7 +25,7 @@ def _log_temporary_tables_creation(
25
25
  """
26
26
  from meerschaum.utils.misc import items_str
27
27
  from meerschaum.connectors.sql.tables import get_tables
28
- sqlalchemy = mrsm.attempt_import('sqlalchemy')
28
+ sqlalchemy = mrsm.attempt_import('sqlalchemy', lazy=False)
29
29
  temp_tables_table = get_tables(
30
30
  mrsm_instance=self,
31
31
  create=create,
@@ -86,7 +86,7 @@ def _drop_temporary_tables(self, debug: bool = False) -> SuccessTuple:
86
86
  """
87
87
  from meerschaum.utils.misc import items_str
88
88
  from meerschaum.connectors.sql.tables import get_tables
89
- sqlalchemy = mrsm.attempt_import('sqlalchemy')
89
+ sqlalchemy = mrsm.attempt_import('sqlalchemy', lazy=False)
90
90
  temp_tables_table = get_tables(
91
91
  mrsm_instance=self,
92
92
  create=False,
@@ -150,7 +150,7 @@ def _drop_old_temporary_tables(
150
150
  """
151
151
  from meerschaum.config import get_config
152
152
  from meerschaum.connectors.sql.tables import get_tables
153
- sqlalchemy = mrsm.attempt_import('sqlalchemy')
153
+ sqlalchemy = mrsm.attempt_import('sqlalchemy', lazy=False)
154
154
  temp_tables_table = get_tables(mrsm_instance=self, create=False, debug=debug)['temp_tables']
155
155
  last_check = getattr(self, '_stale_temporary_tables_check_timestamp', 0)
156
156
  now_ts = time.perf_counter()
@@ -55,7 +55,7 @@ def register_pipe(
55
55
  parameters = {}
56
56
 
57
57
  import json
58
- sqlalchemy = attempt_import('sqlalchemy')
58
+ sqlalchemy = attempt_import('sqlalchemy', lazy=False)
59
59
  values = {
60
60
  'connector_keys' : pipe.connector_keys,
61
61
  'metric_key' : pipe.metric_key,
@@ -118,7 +118,7 @@ def edit_pipe(
118
118
  pipes_tbl = get_tables(mrsm_instance=self, create=(not pipe.temporary), debug=debug)['pipes']
119
119
 
120
120
  import json
121
- sqlalchemy = attempt_import('sqlalchemy')
121
+ sqlalchemy = attempt_import('sqlalchemy', lazy=False)
122
122
 
123
123
  values = {
124
124
  'parameters': (
@@ -176,7 +176,10 @@ def fetch_pipes_keys(
176
176
  from meerschaum.config.static import STATIC_CONFIG
177
177
  import json
178
178
  from copy import deepcopy
179
- sqlalchemy, sqlalchemy_sql_functions = attempt_import('sqlalchemy', 'sqlalchemy.sql.functions')
179
+ sqlalchemy, sqlalchemy_sql_functions = attempt_import(
180
+ 'sqlalchemy',
181
+ 'sqlalchemy.sql.functions', lazy=False,
182
+ )
180
183
  coalesce = sqlalchemy_sql_functions.coalesce
181
184
 
182
185
  if connector_keys is None:
@@ -246,7 +249,7 @@ def fetch_pipes_keys(
246
249
 
247
250
  q = sqlalchemy.select(*select_cols).where(sqlalchemy.and_(True, *_where))
248
251
  for c, vals in cols.items():
249
- if not isinstance(vals, (list, tuple)) or not vals or not c in pipes_tbl.c:
252
+ if not isinstance(vals, (list, tuple)) or not vals or c not in pipes_tbl.c:
250
253
  continue
251
254
  _in_vals, _ex_vals = separate_negation_values(vals)
252
255
  q = q.where(coalesce(pipes_tbl.c[c], 'None').in_(_in_vals)) if _in_vals else q
@@ -306,9 +309,28 @@ def fetch_pipes_keys(
306
309
  return [(row[0], row[1], row[2]) for row in rows]
307
310
 
308
311
 
312
+ def create_pipe_indices(
313
+ self,
314
+ pipe: mrsm.Pipe,
315
+ columns: Optional[List[str]] = None,
316
+ debug: bool = False,
317
+ ) -> SuccessTuple:
318
+ """
319
+ Create a pipe's indices.
320
+ """
321
+ success = self.create_indices(pipe, columns=columns, debug=debug)
322
+ msg = (
323
+ "Success"
324
+ if success
325
+ else f"Failed to create indices for {pipe}."
326
+ )
327
+ return success, msg
328
+
329
+
309
330
  def create_indices(
310
331
  self,
311
332
  pipe: mrsm.Pipe,
333
+ columns: Optional[List[str]] = None,
312
334
  indices: Optional[List[str]] = None,
313
335
  debug: bool = False
314
336
  ) -> bool:
@@ -318,29 +340,51 @@ def create_indices(
318
340
  from meerschaum.utils.debug import dprint
319
341
  if debug:
320
342
  dprint(f"Creating indices for {pipe}...")
343
+
321
344
  if not pipe.indices:
322
345
  warn(f"{pipe} has no index columns; skipping index creation.", stack=False)
323
346
  return True
324
347
 
348
+ cols_to_include = set((columns or []) + (indices or [])) or None
349
+
325
350
  _ = pipe.__dict__.pop('_columns_indices', None)
326
351
  ix_queries = {
327
- ix: queries
328
- for ix, queries in self.get_create_index_queries(pipe, debug=debug).items()
329
- if indices is None or ix in indices
352
+ col: queries
353
+ for col, queries in self.get_create_index_queries(pipe, debug=debug).items()
354
+ if cols_to_include is None or col in cols_to_include
330
355
  }
331
356
  success = True
332
- for ix, queries in ix_queries.items():
357
+ for col, queries in ix_queries.items():
333
358
  ix_success = all(self.exec_queries(queries, debug=debug, silent=False))
334
359
  success = success and ix_success
335
360
  if not ix_success:
336
- warn(f"Failed to create index on column: {ix}")
361
+ warn(f"Failed to create index on column: {col}")
337
362
 
338
363
  return success
339
364
 
340
365
 
366
+ def drop_pipe_indices(
367
+ self,
368
+ pipe: mrsm.Pipe,
369
+ columns: Optional[List[str]] = None,
370
+ debug: bool = False,
371
+ ) -> SuccessTuple:
372
+ """
373
+ Drop a pipe's indices.
374
+ """
375
+ success = self.drop_indices(pipe, columns=columns, debug=debug)
376
+ msg = (
377
+ "Success"
378
+ if success
379
+ else f"Failed to drop indices for {pipe}."
380
+ )
381
+ return success, msg
382
+
383
+
341
384
  def drop_indices(
342
385
  self,
343
386
  pipe: mrsm.Pipe,
387
+ columns: Optional[List[str]] = None,
344
388
  indices: Optional[List[str]] = None,
345
389
  debug: bool = False
346
390
  ) -> bool:
@@ -350,24 +394,81 @@ def drop_indices(
350
394
  from meerschaum.utils.debug import dprint
351
395
  if debug:
352
396
  dprint(f"Dropping indices for {pipe}...")
353
- if not pipe.columns:
354
- warn(f"Unable to drop indices for {pipe} without columns.", stack=False)
397
+
398
+ if not pipe.indices:
399
+ warn(f"No indices to drop for {pipe}.", stack=False)
355
400
  return False
401
+
402
+ cols_to_include = set((columns or []) + (indices or [])) or None
403
+
356
404
  ix_queries = {
357
- ix: queries
358
- for ix, queries in self.get_drop_index_queries(pipe, debug=debug).items()
359
- if indices is None or ix in indices
405
+ col: queries
406
+ for col, queries in self.get_drop_index_queries(pipe, debug=debug).items()
407
+ if cols_to_include is None or col in cols_to_include
360
408
  }
361
409
  success = True
362
- for ix, queries in ix_queries.items():
363
- ix_success = all(self.exec_queries(queries, debug=debug, silent=True))
410
+ for col, queries in ix_queries.items():
411
+ ix_success = all(self.exec_queries(queries, debug=debug, silent=(not debug)))
364
412
  if not ix_success:
365
413
  success = False
366
414
  if debug:
367
- dprint(f"Failed to drop index on column: {ix}")
415
+ dprint(f"Failed to drop index on column: {col}")
368
416
  return success
369
417
 
370
418
 
419
+ def get_pipe_index_names(self, pipe: mrsm.Pipe) -> Dict[str, str]:
420
+ """
421
+ Return a dictionary mapping index keys to their names on the database.
422
+
423
+ Returns
424
+ -------
425
+ A dictionary of index keys to column names.
426
+ """
427
+ from meerschaum.utils.sql import DEFAULT_SCHEMA_FLAVORS
428
+ _parameters = pipe.parameters
429
+ _index_template = _parameters.get('index_template', "IX_{schema_str}{target}_{column_names}")
430
+ _schema = self.get_pipe_schema(pipe)
431
+ if _schema is None:
432
+ _schema = (
433
+ DEFAULT_SCHEMA_FLAVORS.get(self.flavor, None)
434
+ if self.flavor != 'mssql'
435
+ else None
436
+ )
437
+ schema_str = '' if _schema is None else f'{_schema}_'
438
+ schema_str = ''
439
+ _indices = pipe.indices
440
+ _target = pipe.target
441
+ _column_names = {
442
+ ix: (
443
+ '_'.join(cols)
444
+ if isinstance(cols, (list, tuple))
445
+ else str(cols)
446
+ )
447
+ for ix, cols in _indices.items()
448
+ if cols
449
+ }
450
+ _index_names = {
451
+ ix: _index_template.format(
452
+ target=_target,
453
+ column_names=column_names,
454
+ connector_keys=pipe.connector_keys,
455
+ metric_key=pipe.metric_key,
456
+ location_key=pipe.location_key,
457
+ schema_str=schema_str,
458
+ )
459
+ for ix, column_names in _column_names.items()
460
+ }
461
+ ### NOTE: Skip any duplicate indices.
462
+ seen_index_names = {}
463
+ for ix, index_name in _index_names.items():
464
+ if index_name in seen_index_names:
465
+ continue
466
+ seen_index_names[index_name] = ix
467
+ return {
468
+ ix: index_name
469
+ for index_name, ix in seen_index_names.items()
470
+ }
471
+
371
472
  def get_create_index_queries(
372
473
  self,
373
474
  pipe: mrsm.Pipe,
@@ -407,7 +508,11 @@ def get_create_index_queries(
407
508
 
408
509
  upsert = pipe.parameters.get('upsert', False) and (self.flavor + '-upsert') in UPDATE_QUERIES
409
510
  static = pipe.parameters.get('static', False)
511
+ null_indices = pipe.parameters.get('null_indices', True)
410
512
  index_names = pipe.get_indices()
513
+ unique_index_name_unquoted = index_names.get('unique', None) or f'IX_{pipe.target}_unique'
514
+ if upsert:
515
+ _ = index_names.pop('unique', None)
411
516
  indices = pipe.indices
412
517
  existing_cols_types = pipe.get_columns_types(debug=debug)
413
518
  existing_cols_pd_types = {
@@ -420,11 +525,11 @@ def get_create_index_queries(
420
525
  existing_clustered_primary_keys = []
421
526
  for col, col_indices in existing_cols_indices.items():
422
527
  for col_ix_doc in col_indices:
423
- existing_ix_names.add(col_ix_doc.get('name', None))
528
+ existing_ix_names.add(col_ix_doc.get('name', '').lower())
424
529
  if col_ix_doc.get('type', None) == 'PRIMARY KEY':
425
- existing_primary_keys.append(col)
530
+ existing_primary_keys.append(col.lower())
426
531
  if col_ix_doc.get('clustered', True):
427
- existing_clustered_primary_keys.append(col)
532
+ existing_clustered_primary_keys.append(col.lower())
428
533
 
429
534
  _datetime = pipe.get_columns('datetime', error=False)
430
535
  _datetime_name = (
@@ -456,7 +561,7 @@ def get_create_index_queries(
456
561
  )
457
562
  )
458
563
  primary_key_db_type = (
459
- get_db_type_from_pd_type(pipe.dtypes.get(primary_key, 'int'), self.flavor)
564
+ get_db_type_from_pd_type(pipe.dtypes.get(primary_key, 'int') or 'int', self.flavor)
460
565
  if primary_key
461
566
  else None
462
567
  )
@@ -471,6 +576,19 @@ def get_create_index_queries(
471
576
  if not existing_clustered_primary_keys and _datetime is not None
472
577
  else "NONCLUSTERED"
473
578
  )
579
+ include_columns_str = "\n ,".join(
580
+ [
581
+ sql_item_name(col, flavor=self.flavor) for col in existing_cols_types
582
+ if col != _datetime
583
+ ]
584
+ ).rstrip(',')
585
+ include_clause = (
586
+ (
587
+ f"\nINCLUDE (\n {include_columns_str}\n)"
588
+ )
589
+ if datetime_clustered == 'NONCLUSTERED'
590
+ else ''
591
+ )
474
592
 
475
593
  _id_index_name = (
476
594
  sql_item_name(index_names['id'], self.flavor, None)
@@ -516,7 +634,7 @@ def get_create_index_queries(
516
634
  if self.flavor == 'mssql':
517
635
  dt_query = (
518
636
  f"CREATE {datetime_clustered} INDEX {_datetime_index_name} "
519
- f"ON {_pipe_name} ({_datetime_name})"
637
+ f"\nON {_pipe_name} ({_datetime_name}){include_clause}"
520
638
  )
521
639
  else:
522
640
  dt_query = (
@@ -530,7 +648,7 @@ def get_create_index_queries(
530
648
  primary_queries = []
531
649
  if (
532
650
  primary_key is not None
533
- and primary_key not in existing_primary_keys
651
+ and primary_key.lower() not in existing_primary_keys
534
652
  and not static
535
653
  ):
536
654
  if autoincrement and primary_key not in existing_cols_pd_types:
@@ -659,7 +777,10 @@ def get_create_index_queries(
659
777
  other_index_names = {
660
778
  ix_key: ix_unquoted
661
779
  for ix_key, ix_unquoted in index_names.items()
662
- if ix_key not in ('datetime', 'id', 'primary') and ix_unquoted not in existing_ix_names
780
+ if (
781
+ ix_key not in ('datetime', 'id', 'primary')
782
+ and ix_unquoted.lower() not in existing_ix_names
783
+ )
663
784
  }
664
785
  for ix_key, ix_unquoted in other_index_names.items():
665
786
  ix_name = sql_item_name(ix_unquoted, self.flavor, None)
@@ -684,24 +805,29 @@ def get_create_index_queries(
684
805
  coalesce_indices_cols_str = ', '.join(
685
806
  [
686
807
  (
687
- "COALESCE("
688
- + sql_item_name(ix, self.flavor)
689
- + ", "
690
- + get_null_replacement(existing_cols_types[ix], self.flavor)
691
- + ") "
692
- ) if ix_key != 'datetime' else (sql_item_name(ix, self.flavor))
808
+ (
809
+ "COALESCE("
810
+ + sql_item_name(ix, self.flavor)
811
+ + ", "
812
+ + get_null_replacement(existing_cols_types[ix], self.flavor)
813
+ + ") "
814
+ )
815
+ if ix_key != 'datetime' and null_indices
816
+ else sql_item_name(ix, self.flavor)
817
+ )
693
818
  for ix_key, ix in pipe.columns.items()
694
819
  if ix and ix in existing_cols_types
695
820
  ]
696
821
  )
697
- unique_index_name = sql_item_name(pipe.target + '_unique_index', self.flavor)
698
- constraint_name = sql_item_name(pipe.target + '_constraint', self.flavor)
822
+ unique_index_name = sql_item_name(unique_index_name_unquoted, self.flavor)
823
+ constraint_name_unquoted = unique_index_name_unquoted.replace('IX_', 'UQ_')
824
+ constraint_name = sql_item_name(constraint_name_unquoted, self.flavor)
699
825
  add_constraint_query = (
700
826
  f"ALTER TABLE {_pipe_name} ADD CONSTRAINT {constraint_name} UNIQUE ({indices_cols_str})"
701
827
  )
702
828
  unique_index_cols_str = (
703
829
  indices_cols_str
704
- if self.flavor not in COALESCE_UNIQUE_INDEX_FLAVORS
830
+ if self.flavor not in COALESCE_UNIQUE_INDEX_FLAVORS or not null_indices
705
831
  else coalesce_indices_cols_str
706
832
  )
707
833
  create_unique_index_query = (
@@ -737,21 +863,33 @@ def get_drop_index_queries(
737
863
  return {}
738
864
  if not pipe.exists(debug=debug):
739
865
  return {}
866
+
867
+ from collections import defaultdict
740
868
  from meerschaum.utils.sql import (
741
869
  sql_item_name,
742
870
  table_exists,
743
871
  hypertable_queries,
744
- DROP_IF_EXISTS_FLAVORS,
872
+ DROP_INDEX_IF_EXISTS_FLAVORS,
745
873
  )
746
- drop_queries = {}
874
+ drop_queries = defaultdict(lambda: [])
747
875
  schema = self.get_pipe_schema(pipe)
748
- schema_prefix = (schema + '_') if schema else ''
876
+ index_schema = schema if self.flavor != 'mssql' else None
749
877
  indices = {
750
- col: schema_prefix + ix
751
- for col, ix in pipe.get_indices().items()
878
+ ix_key: ix
879
+ for ix_key, ix in pipe.get_indices().items()
752
880
  }
753
- pipe_name = sql_item_name(pipe.target, self.flavor, self.get_pipe_schema(pipe))
881
+ cols_indices = pipe.get_columns_indices(debug=debug)
882
+ existing_indices = set()
883
+ clustered_ix = None
884
+ for col, ix_metas in cols_indices.items():
885
+ for ix_meta in ix_metas:
886
+ ix_name = ix_meta.get('name', None)
887
+ if ix_meta.get('clustered', False):
888
+ clustered_ix = ix_name
889
+ existing_indices.add(ix_name.lower())
890
+ pipe_name = sql_item_name(pipe.target, self.flavor, schema)
754
891
  pipe_name_no_schema = sql_item_name(pipe.target, self.flavor, None)
892
+ upsert = pipe.upsert
755
893
 
756
894
  if self.flavor not in hypertable_queries:
757
895
  is_hypertable = False
@@ -759,7 +897,7 @@ def get_drop_index_queries(
759
897
  is_hypertable_query = hypertable_queries[self.flavor].format(table_name=pipe_name)
760
898
  is_hypertable = self.value(is_hypertable_query, silent=True, debug=debug) is not None
761
899
 
762
- if_exists_str = "IF EXISTS" if self.flavor in DROP_IF_EXISTS_FLAVORS else ""
900
+ if_exists_str = "IF EXISTS " if self.flavor in DROP_INDEX_IF_EXISTS_FLAVORS else ""
763
901
  if is_hypertable:
764
902
  nuke_queries = []
765
903
  temp_table = '_' + pipe.target + '_temp_migration'
@@ -769,21 +907,51 @@ def get_drop_index_queries(
769
907
  nuke_queries.append(f"DROP TABLE {if_exists_str} {temp_table_name}")
770
908
  nuke_queries += [
771
909
  f"SELECT * INTO {temp_table_name} FROM {pipe_name}",
772
- f"DROP TABLE {if_exists_str} {pipe_name}",
910
+ f"DROP TABLE {if_exists_str}{pipe_name}",
773
911
  f"ALTER TABLE {temp_table_name} RENAME TO {pipe_name_no_schema}",
774
912
  ]
775
913
  nuke_ix_keys = ('datetime', 'id')
776
914
  nuked = False
777
915
  for ix_key in nuke_ix_keys:
778
916
  if ix_key in indices and not nuked:
779
- drop_queries[ix_key] = nuke_queries
917
+ drop_queries[ix_key].extend(nuke_queries)
780
918
  nuked = True
781
919
 
782
- drop_queries.update({
783
- ix_key: ["DROP INDEX " + sql_item_name(ix_unquoted, self.flavor, None)]
784
- for ix_key, ix_unquoted in indices.items()
785
- if ix_key not in drop_queries
786
- })
920
+ for ix_key, ix_unquoted in indices.items():
921
+ if ix_key in drop_queries:
922
+ continue
923
+ if ix_unquoted.lower() not in existing_indices:
924
+ continue
925
+
926
+ if ix_key == 'unique' and upsert and self.flavor not in ('sqlite',) and not is_hypertable:
927
+ constraint_name_unquoted = ix_unquoted.replace('IX_', 'UQ_')
928
+ constraint_name = sql_item_name(constraint_name_unquoted, self.flavor)
929
+ constraint_or_index = (
930
+ "CONSTRAINT"
931
+ if self.flavor not in ('mysql', 'mariadb')
932
+ else 'INDEX'
933
+ )
934
+ drop_queries[ix_key].append(
935
+ f"ALTER TABLE {pipe_name}\n"
936
+ f"DROP {constraint_or_index} {constraint_name}"
937
+ )
938
+
939
+ query = (
940
+ (
941
+ f"ALTER TABLE {pipe_name}\n"
942
+ if self.flavor in ('mysql', 'mariadb')
943
+ else ''
944
+ )
945
+ + f"DROP INDEX {if_exists_str}"
946
+ + sql_item_name(ix_unquoted, self.flavor, index_schema)
947
+ )
948
+ if self.flavor == 'mssql':
949
+ query += f"\nON {pipe_name}"
950
+ if ix_unquoted == clustered_ix:
951
+ query += "\nWITH (ONLINE = ON, MAXDOP = 4)"
952
+ drop_queries[ix_key].append(query)
953
+
954
+
787
955
  return drop_queries
788
956
 
789
957
 
@@ -796,7 +964,7 @@ def delete_pipe(
796
964
  Delete a Pipe's registration.
797
965
  """
798
966
  from meerschaum.utils.packages import attempt_import
799
- sqlalchemy = attempt_import('sqlalchemy')
967
+ sqlalchemy = attempt_import('sqlalchemy', lazy=False)
800
968
 
801
969
  if not pipe.id:
802
970
  return False, f"{pipe} is not registered."
@@ -1368,7 +1536,7 @@ def create_pipe_table_from_df(
1368
1536
  from meerschaum.utils.dtypes.sql import get_db_type_from_pd_type
1369
1537
  primary_key = pipe.columns.get('primary', None)
1370
1538
  primary_key_typ = (
1371
- pipe.dtypes.get(primary_key, str(df.dtypes.get(primary_key)))
1539
+ pipe.dtypes.get(primary_key, str(df.dtypes.get(primary_key, 'int')))
1372
1540
  if primary_key
1373
1541
  else None
1374
1542
  )
@@ -1777,6 +1945,7 @@ def sync_pipe(
1777
1945
  patch_schema=self.internal_schema,
1778
1946
  datetime_col=(dt_col if dt_col in update_df.columns else None),
1779
1947
  identity_insert=(autoincrement and primary_key in update_df.columns),
1948
+ null_indices=pipe.null_indices,
1780
1949
  debug=debug,
1781
1950
  )
1782
1951
  update_results = self.exec_queries(
@@ -2077,13 +2246,19 @@ def sync_pipe_inplace(
2077
2246
  _ = clean_up_temp_tables()
2078
2247
  return True, f"Inserted {new_count}, updated 0 rows."
2079
2248
 
2080
- dt_col_name_da = dateadd_str(flavor=self.flavor, begin=dt_col_name, db_type=dt_db_type)
2249
+ min_dt_col_name_da = dateadd_str(
2250
+ flavor=self.flavor, begin=f"MIN({dt_col_name})", db_type=dt_db_type,
2251
+ )
2252
+ max_dt_col_name_da = dateadd_str(
2253
+ flavor=self.flavor, begin=f"MAX({dt_col_name})", db_type=dt_db_type,
2254
+ )
2255
+
2081
2256
  (new_dt_bounds_success, new_dt_bounds_msg), new_dt_bounds_results = session_execute(
2082
2257
  session,
2083
2258
  [
2084
2259
  "SELECT\n"
2085
- f" MIN({dt_col_name_da}) AS {sql_item_name('min_dt', self.flavor)},\n"
2086
- f" MAX({dt_col_name_da}) AS {sql_item_name('max_dt', self.flavor)}\n"
2260
+ f" {min_dt_col_name_da} AS {sql_item_name('min_dt', self.flavor)},\n"
2261
+ f" {max_dt_col_name_da} AS {sql_item_name('max_dt', self.flavor)}\n"
2087
2262
  f"FROM {temp_table_names['new' if not upsert else 'update']}\n"
2088
2263
  f"WHERE {dt_col_name} IS NOT NULL"
2089
2264
  ],
@@ -2350,6 +2525,7 @@ def sync_pipe_inplace(
2350
2525
  patch_schema=internal_schema,
2351
2526
  datetime_col=pipe.columns.get('datetime', None),
2352
2527
  flavor=self.flavor,
2528
+ null_indices=pipe.null_indices,
2353
2529
  debug=debug,
2354
2530
  )
2355
2531
  if on_cols else []
@@ -2643,7 +2819,7 @@ def get_pipe_rowcount(
2643
2819
  _cols_names = ['*']
2644
2820
 
2645
2821
  src = (
2646
- f"SELECT {', '.join(_cols_names)} FROM {_pipe_name}"
2822
+ f"SELECT {', '.join(_cols_names)}\nFROM {_pipe_name}"
2647
2823
  if not remote
2648
2824
  else get_pipe_query(pipe)
2649
2825
  )
@@ -2652,15 +2828,17 @@ def get_pipe_rowcount(
2652
2828
  if begin is not None or end is not None:
2653
2829
  query += "\nWHERE"
2654
2830
  if begin is not None:
2655
- query += f"""
2656
- {dt_name} >= {dateadd_str(self.flavor, datepart='minute', number=0, begin=begin, db_type=dt_db_type)}
2657
- """
2831
+ query += (
2832
+ f"\n {dt_name} >= "
2833
+ + dateadd_str(self.flavor, datepart='minute', number=0, begin=begin, db_type=dt_db_type)
2834
+ )
2658
2835
  if end is not None and begin is not None:
2659
- query += "AND"
2836
+ query += "\n AND"
2660
2837
  if end is not None:
2661
- query += f"""
2662
- {dt_name} < {dateadd_str(self.flavor, datepart='minute', number=0, begin=end, db_type=dt_db_type)}
2663
- """
2838
+ query += (
2839
+ f"\n {dt_name} < "
2840
+ + dateadd_str(self.flavor, datepart='minute', number=0, begin=end, db_type=dt_db_type)
2841
+ )
2664
2842
  if params is not None:
2665
2843
  from meerschaum.utils.sql import build_where
2666
2844
  existing_cols = pipe.get_columns_types(debug=debug)
@@ -2782,13 +2960,21 @@ def clear_pipe(
2782
2960
  valid_params = {k: v for k, v in params.items() if k in existing_cols}
2783
2961
  clear_query = (
2784
2962
  f"DELETE FROM {pipe_name}\nWHERE 1 = 1\n"
2785
- + (' AND ' + build_where(valid_params, self, with_where=False) if valid_params else '')
2963
+ + ('\n AND ' + build_where(valid_params, self, with_where=False) if valid_params else '')
2786
2964
  + (
2787
- f' AND {dt_name} >= ' + dateadd_str(self.flavor, 'day', 0, begin, db_type=dt_db_type)
2788
- if begin is not None else ''
2965
+ (
2966
+ f'\n AND {dt_name} >= '
2967
+ + dateadd_str(self.flavor, 'day', 0, begin, db_type=dt_db_type)
2968
+ )
2969
+ if begin is not None
2970
+ else ''
2789
2971
  ) + (
2790
- f' AND {dt_name} < ' + dateadd_str(self.flavor, 'day', 0, end, db_type=dt_db_type)
2791
- if end is not None else ''
2972
+ (
2973
+ f'\n AND {dt_name} < '
2974
+ + dateadd_str(self.flavor, 'day', 0, end, db_type=dt_db_type)
2975
+ )
2976
+ if end is not None
2977
+ else ''
2792
2978
  )
2793
2979
  )
2794
2980
  success = self.exec(clear_query, silent=True, debug=debug) is not None
@@ -2999,7 +3185,9 @@ def get_add_columns_queries(
2999
3185
  col: get_db_type_from_pd_type(
3000
3186
  df_cols_types[col],
3001
3187
  self.flavor
3002
- ) for col in new_cols
3188
+ )
3189
+ for col in new_cols
3190
+ if col and df_cols_types.get(col, None)
3003
3191
  }
3004
3192
 
3005
3193
  alter_table_query = "ALTER TABLE " + sql_item_name(
@@ -3391,6 +3579,7 @@ def get_to_sql_dtype(
3391
3579
  return {
3392
3580
  col: get_db_type_from_pd_type(typ, self.flavor, as_sqlalchemy=True)
3393
3581
  for col, typ in df_dtypes.items()
3582
+ if col and typ
3394
3583
  }
3395
3584
 
3396
3585