meerschaum 2.7.6__py3-none-any.whl → 2.7.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/actions/copy.py +1 -0
- meerschaum/actions/drop.py +100 -22
- meerschaum/actions/index.py +71 -0
- meerschaum/actions/register.py +8 -12
- meerschaum/actions/sql.py +1 -1
- meerschaum/api/routes/_pipes.py +18 -0
- meerschaum/api/routes/_plugins.py +1 -1
- meerschaum/api/routes/_users.py +62 -61
- meerschaum/config/_version.py +1 -1
- meerschaum/connectors/api/_pipes.py +20 -0
- meerschaum/connectors/sql/_SQLConnector.py +8 -12
- meerschaum/connectors/sql/_create_engine.py +1 -1
- meerschaum/connectors/sql/_fetch.py +9 -39
- meerschaum/connectors/sql/_instance.py +3 -3
- meerschaum/connectors/sql/_pipes.py +262 -70
- meerschaum/connectors/sql/_plugins.py +11 -16
- meerschaum/connectors/sql/_sql.py +60 -39
- meerschaum/connectors/sql/_uri.py +9 -9
- meerschaum/connectors/sql/_users.py +10 -12
- meerschaum/connectors/sql/tables/__init__.py +13 -14
- meerschaum/connectors/valkey/_ValkeyConnector.py +2 -2
- meerschaum/core/Pipe/__init__.py +12 -2
- meerschaum/core/Pipe/_attributes.py +32 -38
- meerschaum/core/Pipe/_drop.py +73 -2
- meerschaum/core/Pipe/_fetch.py +4 -0
- meerschaum/core/Pipe/_index.py +68 -0
- meerschaum/core/Pipe/_sync.py +16 -9
- meerschaum/utils/daemon/Daemon.py +9 -2
- meerschaum/utils/daemon/RotatingFile.py +3 -3
- meerschaum/utils/dataframe.py +42 -12
- meerschaum/utils/dtypes/__init__.py +144 -24
- meerschaum/utils/dtypes/sql.py +52 -9
- meerschaum/utils/formatting/__init__.py +2 -2
- meerschaum/utils/formatting/_pprint.py +12 -11
- meerschaum/utils/misc.py +16 -18
- meerschaum/utils/prompt.py +1 -1
- meerschaum/utils/sql.py +106 -42
- {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/METADATA +14 -2
- {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/RECORD +45 -43
- {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/WHEEL +1 -1
- {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/LICENSE +0 -0
- {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/NOTICE +0 -0
- {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/top_level.txt +0 -0
- {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/zip-safe +0 -0
@@ -55,7 +55,7 @@ def register_pipe(
|
|
55
55
|
parameters = {}
|
56
56
|
|
57
57
|
import json
|
58
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
58
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
59
59
|
values = {
|
60
60
|
'connector_keys' : pipe.connector_keys,
|
61
61
|
'metric_key' : pipe.metric_key,
|
@@ -118,7 +118,7 @@ def edit_pipe(
|
|
118
118
|
pipes_tbl = get_tables(mrsm_instance=self, create=(not pipe.temporary), debug=debug)['pipes']
|
119
119
|
|
120
120
|
import json
|
121
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
121
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
122
122
|
|
123
123
|
values = {
|
124
124
|
'parameters': (
|
@@ -176,7 +176,10 @@ def fetch_pipes_keys(
|
|
176
176
|
from meerschaum.config.static import STATIC_CONFIG
|
177
177
|
import json
|
178
178
|
from copy import deepcopy
|
179
|
-
sqlalchemy, sqlalchemy_sql_functions = attempt_import(
|
179
|
+
sqlalchemy, sqlalchemy_sql_functions = attempt_import(
|
180
|
+
'sqlalchemy',
|
181
|
+
'sqlalchemy.sql.functions', lazy=False,
|
182
|
+
)
|
180
183
|
coalesce = sqlalchemy_sql_functions.coalesce
|
181
184
|
|
182
185
|
if connector_keys is None:
|
@@ -246,7 +249,7 @@ def fetch_pipes_keys(
|
|
246
249
|
|
247
250
|
q = sqlalchemy.select(*select_cols).where(sqlalchemy.and_(True, *_where))
|
248
251
|
for c, vals in cols.items():
|
249
|
-
if not isinstance(vals, (list, tuple)) or not vals or not
|
252
|
+
if not isinstance(vals, (list, tuple)) or not vals or c not in pipes_tbl.c:
|
250
253
|
continue
|
251
254
|
_in_vals, _ex_vals = separate_negation_values(vals)
|
252
255
|
q = q.where(coalesce(pipes_tbl.c[c], 'None').in_(_in_vals)) if _in_vals else q
|
@@ -306,9 +309,28 @@ def fetch_pipes_keys(
|
|
306
309
|
return [(row[0], row[1], row[2]) for row in rows]
|
307
310
|
|
308
311
|
|
312
|
+
def create_pipe_indices(
|
313
|
+
self,
|
314
|
+
pipe: mrsm.Pipe,
|
315
|
+
columns: Optional[List[str]] = None,
|
316
|
+
debug: bool = False,
|
317
|
+
) -> SuccessTuple:
|
318
|
+
"""
|
319
|
+
Create a pipe's indices.
|
320
|
+
"""
|
321
|
+
success = self.create_indices(pipe, columns=columns, debug=debug)
|
322
|
+
msg = (
|
323
|
+
"Success"
|
324
|
+
if success
|
325
|
+
else f"Failed to create indices for {pipe}."
|
326
|
+
)
|
327
|
+
return success, msg
|
328
|
+
|
329
|
+
|
309
330
|
def create_indices(
|
310
331
|
self,
|
311
332
|
pipe: mrsm.Pipe,
|
333
|
+
columns: Optional[List[str]] = None,
|
312
334
|
indices: Optional[List[str]] = None,
|
313
335
|
debug: bool = False
|
314
336
|
) -> bool:
|
@@ -318,29 +340,51 @@ def create_indices(
|
|
318
340
|
from meerschaum.utils.debug import dprint
|
319
341
|
if debug:
|
320
342
|
dprint(f"Creating indices for {pipe}...")
|
343
|
+
|
321
344
|
if not pipe.indices:
|
322
345
|
warn(f"{pipe} has no index columns; skipping index creation.", stack=False)
|
323
346
|
return True
|
324
347
|
|
348
|
+
cols_to_include = set((columns or []) + (indices or [])) or None
|
349
|
+
|
325
350
|
_ = pipe.__dict__.pop('_columns_indices', None)
|
326
351
|
ix_queries = {
|
327
|
-
|
328
|
-
for
|
329
|
-
if
|
352
|
+
col: queries
|
353
|
+
for col, queries in self.get_create_index_queries(pipe, debug=debug).items()
|
354
|
+
if cols_to_include is None or col in cols_to_include
|
330
355
|
}
|
331
356
|
success = True
|
332
|
-
for
|
357
|
+
for col, queries in ix_queries.items():
|
333
358
|
ix_success = all(self.exec_queries(queries, debug=debug, silent=False))
|
334
359
|
success = success and ix_success
|
335
360
|
if not ix_success:
|
336
|
-
warn(f"Failed to create index on column: {
|
361
|
+
warn(f"Failed to create index on column: {col}")
|
337
362
|
|
338
363
|
return success
|
339
364
|
|
340
365
|
|
366
|
+
def drop_pipe_indices(
|
367
|
+
self,
|
368
|
+
pipe: mrsm.Pipe,
|
369
|
+
columns: Optional[List[str]] = None,
|
370
|
+
debug: bool = False,
|
371
|
+
) -> SuccessTuple:
|
372
|
+
"""
|
373
|
+
Drop a pipe's indices.
|
374
|
+
"""
|
375
|
+
success = self.drop_indices(pipe, columns=columns, debug=debug)
|
376
|
+
msg = (
|
377
|
+
"Success"
|
378
|
+
if success
|
379
|
+
else f"Failed to drop indices for {pipe}."
|
380
|
+
)
|
381
|
+
return success, msg
|
382
|
+
|
383
|
+
|
341
384
|
def drop_indices(
|
342
385
|
self,
|
343
386
|
pipe: mrsm.Pipe,
|
387
|
+
columns: Optional[List[str]] = None,
|
344
388
|
indices: Optional[List[str]] = None,
|
345
389
|
debug: bool = False
|
346
390
|
) -> bool:
|
@@ -350,24 +394,81 @@ def drop_indices(
|
|
350
394
|
from meerschaum.utils.debug import dprint
|
351
395
|
if debug:
|
352
396
|
dprint(f"Dropping indices for {pipe}...")
|
353
|
-
|
354
|
-
|
397
|
+
|
398
|
+
if not pipe.indices:
|
399
|
+
warn(f"No indices to drop for {pipe}.", stack=False)
|
355
400
|
return False
|
401
|
+
|
402
|
+
cols_to_include = set((columns or []) + (indices or [])) or None
|
403
|
+
|
356
404
|
ix_queries = {
|
357
|
-
|
358
|
-
for
|
359
|
-
if
|
405
|
+
col: queries
|
406
|
+
for col, queries in self.get_drop_index_queries(pipe, debug=debug).items()
|
407
|
+
if cols_to_include is None or col in cols_to_include
|
360
408
|
}
|
361
409
|
success = True
|
362
|
-
for
|
363
|
-
ix_success = all(self.exec_queries(queries, debug=debug, silent=
|
410
|
+
for col, queries in ix_queries.items():
|
411
|
+
ix_success = all(self.exec_queries(queries, debug=debug, silent=(not debug)))
|
364
412
|
if not ix_success:
|
365
413
|
success = False
|
366
414
|
if debug:
|
367
|
-
dprint(f"Failed to drop index on column: {
|
415
|
+
dprint(f"Failed to drop index on column: {col}")
|
368
416
|
return success
|
369
417
|
|
370
418
|
|
419
|
+
def get_pipe_index_names(self, pipe: mrsm.Pipe) -> Dict[str, str]:
|
420
|
+
"""
|
421
|
+
Return a dictionary mapping index keys to their names on the database.
|
422
|
+
|
423
|
+
Returns
|
424
|
+
-------
|
425
|
+
A dictionary of index keys to column names.
|
426
|
+
"""
|
427
|
+
from meerschaum.utils.sql import DEFAULT_SCHEMA_FLAVORS
|
428
|
+
_parameters = pipe.parameters
|
429
|
+
_index_template = _parameters.get('index_template', "IX_{schema_str}{target}_{column_names}")
|
430
|
+
_schema = self.get_pipe_schema(pipe)
|
431
|
+
if _schema is None:
|
432
|
+
_schema = (
|
433
|
+
DEFAULT_SCHEMA_FLAVORS.get(self.flavor, None)
|
434
|
+
if self.flavor != 'mssql'
|
435
|
+
else None
|
436
|
+
)
|
437
|
+
schema_str = '' if _schema is None else f'{_schema}_'
|
438
|
+
schema_str = ''
|
439
|
+
_indices = pipe.indices
|
440
|
+
_target = pipe.target
|
441
|
+
_column_names = {
|
442
|
+
ix: (
|
443
|
+
'_'.join(cols)
|
444
|
+
if isinstance(cols, (list, tuple))
|
445
|
+
else str(cols)
|
446
|
+
)
|
447
|
+
for ix, cols in _indices.items()
|
448
|
+
if cols
|
449
|
+
}
|
450
|
+
_index_names = {
|
451
|
+
ix: _index_template.format(
|
452
|
+
target=_target,
|
453
|
+
column_names=column_names,
|
454
|
+
connector_keys=pipe.connector_keys,
|
455
|
+
metric_key=pipe.metric_key,
|
456
|
+
location_key=pipe.location_key,
|
457
|
+
schema_str=schema_str,
|
458
|
+
)
|
459
|
+
for ix, column_names in _column_names.items()
|
460
|
+
}
|
461
|
+
### NOTE: Skip any duplicate indices.
|
462
|
+
seen_index_names = {}
|
463
|
+
for ix, index_name in _index_names.items():
|
464
|
+
if index_name in seen_index_names:
|
465
|
+
continue
|
466
|
+
seen_index_names[index_name] = ix
|
467
|
+
return {
|
468
|
+
ix: index_name
|
469
|
+
for index_name, ix in seen_index_names.items()
|
470
|
+
}
|
471
|
+
|
371
472
|
def get_create_index_queries(
|
372
473
|
self,
|
373
474
|
pipe: mrsm.Pipe,
|
@@ -407,7 +508,11 @@ def get_create_index_queries(
|
|
407
508
|
|
408
509
|
upsert = pipe.parameters.get('upsert', False) and (self.flavor + '-upsert') in UPDATE_QUERIES
|
409
510
|
static = pipe.parameters.get('static', False)
|
511
|
+
null_indices = pipe.parameters.get('null_indices', True)
|
410
512
|
index_names = pipe.get_indices()
|
513
|
+
unique_index_name_unquoted = index_names.get('unique', None) or f'IX_{pipe.target}_unique'
|
514
|
+
if upsert:
|
515
|
+
_ = index_names.pop('unique', None)
|
411
516
|
indices = pipe.indices
|
412
517
|
existing_cols_types = pipe.get_columns_types(debug=debug)
|
413
518
|
existing_cols_pd_types = {
|
@@ -420,11 +525,11 @@ def get_create_index_queries(
|
|
420
525
|
existing_clustered_primary_keys = []
|
421
526
|
for col, col_indices in existing_cols_indices.items():
|
422
527
|
for col_ix_doc in col_indices:
|
423
|
-
existing_ix_names.add(col_ix_doc.get('name',
|
528
|
+
existing_ix_names.add(col_ix_doc.get('name', '').lower())
|
424
529
|
if col_ix_doc.get('type', None) == 'PRIMARY KEY':
|
425
|
-
existing_primary_keys.append(col)
|
530
|
+
existing_primary_keys.append(col.lower())
|
426
531
|
if col_ix_doc.get('clustered', True):
|
427
|
-
existing_clustered_primary_keys.append(col)
|
532
|
+
existing_clustered_primary_keys.append(col.lower())
|
428
533
|
|
429
534
|
_datetime = pipe.get_columns('datetime', error=False)
|
430
535
|
_datetime_name = (
|
@@ -456,7 +561,7 @@ def get_create_index_queries(
|
|
456
561
|
)
|
457
562
|
)
|
458
563
|
primary_key_db_type = (
|
459
|
-
get_db_type_from_pd_type(pipe.dtypes.get(primary_key, 'int'), self.flavor)
|
564
|
+
get_db_type_from_pd_type(pipe.dtypes.get(primary_key, 'int') or 'int', self.flavor)
|
460
565
|
if primary_key
|
461
566
|
else None
|
462
567
|
)
|
@@ -471,6 +576,19 @@ def get_create_index_queries(
|
|
471
576
|
if not existing_clustered_primary_keys and _datetime is not None
|
472
577
|
else "NONCLUSTERED"
|
473
578
|
)
|
579
|
+
include_columns_str = "\n ,".join(
|
580
|
+
[
|
581
|
+
sql_item_name(col, flavor=self.flavor) for col in existing_cols_types
|
582
|
+
if col != _datetime
|
583
|
+
]
|
584
|
+
).rstrip(',')
|
585
|
+
include_clause = (
|
586
|
+
(
|
587
|
+
f"\nINCLUDE (\n {include_columns_str}\n)"
|
588
|
+
)
|
589
|
+
if datetime_clustered == 'NONCLUSTERED'
|
590
|
+
else ''
|
591
|
+
)
|
474
592
|
|
475
593
|
_id_index_name = (
|
476
594
|
sql_item_name(index_names['id'], self.flavor, None)
|
@@ -516,7 +634,7 @@ def get_create_index_queries(
|
|
516
634
|
if self.flavor == 'mssql':
|
517
635
|
dt_query = (
|
518
636
|
f"CREATE {datetime_clustered} INDEX {_datetime_index_name} "
|
519
|
-
f"
|
637
|
+
f"\nON {_pipe_name} ({_datetime_name}){include_clause}"
|
520
638
|
)
|
521
639
|
else:
|
522
640
|
dt_query = (
|
@@ -530,7 +648,7 @@ def get_create_index_queries(
|
|
530
648
|
primary_queries = []
|
531
649
|
if (
|
532
650
|
primary_key is not None
|
533
|
-
and primary_key not in existing_primary_keys
|
651
|
+
and primary_key.lower() not in existing_primary_keys
|
534
652
|
and not static
|
535
653
|
):
|
536
654
|
if autoincrement and primary_key not in existing_cols_pd_types:
|
@@ -659,7 +777,10 @@ def get_create_index_queries(
|
|
659
777
|
other_index_names = {
|
660
778
|
ix_key: ix_unquoted
|
661
779
|
for ix_key, ix_unquoted in index_names.items()
|
662
|
-
if
|
780
|
+
if (
|
781
|
+
ix_key not in ('datetime', 'id', 'primary')
|
782
|
+
and ix_unquoted.lower() not in existing_ix_names
|
783
|
+
)
|
663
784
|
}
|
664
785
|
for ix_key, ix_unquoted in other_index_names.items():
|
665
786
|
ix_name = sql_item_name(ix_unquoted, self.flavor, None)
|
@@ -684,24 +805,29 @@ def get_create_index_queries(
|
|
684
805
|
coalesce_indices_cols_str = ', '.join(
|
685
806
|
[
|
686
807
|
(
|
687
|
-
|
688
|
-
|
689
|
-
|
690
|
-
|
691
|
-
|
692
|
-
|
808
|
+
(
|
809
|
+
"COALESCE("
|
810
|
+
+ sql_item_name(ix, self.flavor)
|
811
|
+
+ ", "
|
812
|
+
+ get_null_replacement(existing_cols_types[ix], self.flavor)
|
813
|
+
+ ") "
|
814
|
+
)
|
815
|
+
if ix_key != 'datetime' and null_indices
|
816
|
+
else sql_item_name(ix, self.flavor)
|
817
|
+
)
|
693
818
|
for ix_key, ix in pipe.columns.items()
|
694
819
|
if ix and ix in existing_cols_types
|
695
820
|
]
|
696
821
|
)
|
697
|
-
unique_index_name = sql_item_name(
|
698
|
-
|
822
|
+
unique_index_name = sql_item_name(unique_index_name_unquoted, self.flavor)
|
823
|
+
constraint_name_unquoted = unique_index_name_unquoted.replace('IX_', 'UQ_')
|
824
|
+
constraint_name = sql_item_name(constraint_name_unquoted, self.flavor)
|
699
825
|
add_constraint_query = (
|
700
826
|
f"ALTER TABLE {_pipe_name} ADD CONSTRAINT {constraint_name} UNIQUE ({indices_cols_str})"
|
701
827
|
)
|
702
828
|
unique_index_cols_str = (
|
703
829
|
indices_cols_str
|
704
|
-
if self.flavor not in COALESCE_UNIQUE_INDEX_FLAVORS
|
830
|
+
if self.flavor not in COALESCE_UNIQUE_INDEX_FLAVORS or not null_indices
|
705
831
|
else coalesce_indices_cols_str
|
706
832
|
)
|
707
833
|
create_unique_index_query = (
|
@@ -737,21 +863,33 @@ def get_drop_index_queries(
|
|
737
863
|
return {}
|
738
864
|
if not pipe.exists(debug=debug):
|
739
865
|
return {}
|
866
|
+
|
867
|
+
from collections import defaultdict
|
740
868
|
from meerschaum.utils.sql import (
|
741
869
|
sql_item_name,
|
742
870
|
table_exists,
|
743
871
|
hypertable_queries,
|
744
|
-
|
872
|
+
DROP_INDEX_IF_EXISTS_FLAVORS,
|
745
873
|
)
|
746
|
-
drop_queries =
|
874
|
+
drop_queries = defaultdict(lambda: [])
|
747
875
|
schema = self.get_pipe_schema(pipe)
|
748
|
-
|
876
|
+
index_schema = schema if self.flavor != 'mssql' else None
|
749
877
|
indices = {
|
750
|
-
|
751
|
-
for
|
878
|
+
ix_key: ix
|
879
|
+
for ix_key, ix in pipe.get_indices().items()
|
752
880
|
}
|
753
|
-
|
881
|
+
cols_indices = pipe.get_columns_indices(debug=debug)
|
882
|
+
existing_indices = set()
|
883
|
+
clustered_ix = None
|
884
|
+
for col, ix_metas in cols_indices.items():
|
885
|
+
for ix_meta in ix_metas:
|
886
|
+
ix_name = ix_meta.get('name', None)
|
887
|
+
if ix_meta.get('clustered', False):
|
888
|
+
clustered_ix = ix_name
|
889
|
+
existing_indices.add(ix_name.lower())
|
890
|
+
pipe_name = sql_item_name(pipe.target, self.flavor, schema)
|
754
891
|
pipe_name_no_schema = sql_item_name(pipe.target, self.flavor, None)
|
892
|
+
upsert = pipe.upsert
|
755
893
|
|
756
894
|
if self.flavor not in hypertable_queries:
|
757
895
|
is_hypertable = False
|
@@ -759,7 +897,7 @@ def get_drop_index_queries(
|
|
759
897
|
is_hypertable_query = hypertable_queries[self.flavor].format(table_name=pipe_name)
|
760
898
|
is_hypertable = self.value(is_hypertable_query, silent=True, debug=debug) is not None
|
761
899
|
|
762
|
-
if_exists_str = "IF EXISTS" if self.flavor in
|
900
|
+
if_exists_str = "IF EXISTS " if self.flavor in DROP_INDEX_IF_EXISTS_FLAVORS else ""
|
763
901
|
if is_hypertable:
|
764
902
|
nuke_queries = []
|
765
903
|
temp_table = '_' + pipe.target + '_temp_migration'
|
@@ -769,21 +907,51 @@ def get_drop_index_queries(
|
|
769
907
|
nuke_queries.append(f"DROP TABLE {if_exists_str} {temp_table_name}")
|
770
908
|
nuke_queries += [
|
771
909
|
f"SELECT * INTO {temp_table_name} FROM {pipe_name}",
|
772
|
-
f"DROP TABLE {if_exists_str}
|
910
|
+
f"DROP TABLE {if_exists_str}{pipe_name}",
|
773
911
|
f"ALTER TABLE {temp_table_name} RENAME TO {pipe_name_no_schema}",
|
774
912
|
]
|
775
913
|
nuke_ix_keys = ('datetime', 'id')
|
776
914
|
nuked = False
|
777
915
|
for ix_key in nuke_ix_keys:
|
778
916
|
if ix_key in indices and not nuked:
|
779
|
-
drop_queries[ix_key]
|
917
|
+
drop_queries[ix_key].extend(nuke_queries)
|
780
918
|
nuked = True
|
781
919
|
|
782
|
-
|
783
|
-
ix_key
|
784
|
-
|
785
|
-
if
|
786
|
-
|
920
|
+
for ix_key, ix_unquoted in indices.items():
|
921
|
+
if ix_key in drop_queries:
|
922
|
+
continue
|
923
|
+
if ix_unquoted.lower() not in existing_indices:
|
924
|
+
continue
|
925
|
+
|
926
|
+
if ix_key == 'unique' and upsert and self.flavor not in ('sqlite',) and not is_hypertable:
|
927
|
+
constraint_name_unquoted = ix_unquoted.replace('IX_', 'UQ_')
|
928
|
+
constraint_name = sql_item_name(constraint_name_unquoted, self.flavor)
|
929
|
+
constraint_or_index = (
|
930
|
+
"CONSTRAINT"
|
931
|
+
if self.flavor not in ('mysql', 'mariadb')
|
932
|
+
else 'INDEX'
|
933
|
+
)
|
934
|
+
drop_queries[ix_key].append(
|
935
|
+
f"ALTER TABLE {pipe_name}\n"
|
936
|
+
f"DROP {constraint_or_index} {constraint_name}"
|
937
|
+
)
|
938
|
+
|
939
|
+
query = (
|
940
|
+
(
|
941
|
+
f"ALTER TABLE {pipe_name}\n"
|
942
|
+
if self.flavor in ('mysql', 'mariadb')
|
943
|
+
else ''
|
944
|
+
)
|
945
|
+
+ f"DROP INDEX {if_exists_str}"
|
946
|
+
+ sql_item_name(ix_unquoted, self.flavor, index_schema)
|
947
|
+
)
|
948
|
+
if self.flavor == 'mssql':
|
949
|
+
query += f"\nON {pipe_name}"
|
950
|
+
if ix_unquoted == clustered_ix:
|
951
|
+
query += "\nWITH (ONLINE = ON, MAXDOP = 4)"
|
952
|
+
drop_queries[ix_key].append(query)
|
953
|
+
|
954
|
+
|
787
955
|
return drop_queries
|
788
956
|
|
789
957
|
|
@@ -796,7 +964,7 @@ def delete_pipe(
|
|
796
964
|
Delete a Pipe's registration.
|
797
965
|
"""
|
798
966
|
from meerschaum.utils.packages import attempt_import
|
799
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
967
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
800
968
|
|
801
969
|
if not pipe.id:
|
802
970
|
return False, f"{pipe} is not registered."
|
@@ -957,7 +1125,7 @@ def get_pipe_data(
|
|
957
1125
|
numeric_columns = [
|
958
1126
|
col
|
959
1127
|
for col, typ in pipe.dtypes.items()
|
960
|
-
if typ
|
1128
|
+
if typ.startswith('numeric') and col in dtypes
|
961
1129
|
]
|
962
1130
|
uuid_columns = [
|
963
1131
|
col
|
@@ -1368,7 +1536,7 @@ def create_pipe_table_from_df(
|
|
1368
1536
|
from meerschaum.utils.dtypes.sql import get_db_type_from_pd_type
|
1369
1537
|
primary_key = pipe.columns.get('primary', None)
|
1370
1538
|
primary_key_typ = (
|
1371
|
-
pipe.dtypes.get(primary_key, str(df.dtypes.get(primary_key)))
|
1539
|
+
pipe.dtypes.get(primary_key, str(df.dtypes.get(primary_key, 'int')))
|
1372
1540
|
if primary_key
|
1373
1541
|
else None
|
1374
1542
|
)
|
@@ -1719,7 +1887,10 @@ def sync_pipe(
|
|
1719
1887
|
warn(f"Could not reset auto-incrementing primary key for {pipe}.", stack=False)
|
1720
1888
|
|
1721
1889
|
if update_df is not None and len(update_df) > 0:
|
1722
|
-
temp_target = self.get_temporary_target(
|
1890
|
+
temp_target = self.get_temporary_target(
|
1891
|
+
pipe.target,
|
1892
|
+
label=('update' if not upsert else 'upsert'),
|
1893
|
+
)
|
1723
1894
|
self._log_temporary_tables_creation(temp_target, create=(not pipe.temporary), debug=debug)
|
1724
1895
|
temp_pipe = Pipe(
|
1725
1896
|
pipe.connector_keys.replace(':', '_') + '_', pipe.metric_key, pipe.location_key,
|
@@ -1777,6 +1948,7 @@ def sync_pipe(
|
|
1777
1948
|
patch_schema=self.internal_schema,
|
1778
1949
|
datetime_col=(dt_col if dt_col in update_df.columns else None),
|
1779
1950
|
identity_insert=(autoincrement and primary_key in update_df.columns),
|
1951
|
+
null_indices=pipe.null_indices,
|
1780
1952
|
debug=debug,
|
1781
1953
|
)
|
1782
1954
|
update_results = self.exec_queries(
|
@@ -2077,13 +2249,19 @@ def sync_pipe_inplace(
|
|
2077
2249
|
_ = clean_up_temp_tables()
|
2078
2250
|
return True, f"Inserted {new_count}, updated 0 rows."
|
2079
2251
|
|
2080
|
-
|
2252
|
+
min_dt_col_name_da = dateadd_str(
|
2253
|
+
flavor=self.flavor, begin=f"MIN({dt_col_name})", db_type=dt_db_type,
|
2254
|
+
)
|
2255
|
+
max_dt_col_name_da = dateadd_str(
|
2256
|
+
flavor=self.flavor, begin=f"MAX({dt_col_name})", db_type=dt_db_type,
|
2257
|
+
)
|
2258
|
+
|
2081
2259
|
(new_dt_bounds_success, new_dt_bounds_msg), new_dt_bounds_results = session_execute(
|
2082
2260
|
session,
|
2083
2261
|
[
|
2084
2262
|
"SELECT\n"
|
2085
|
-
f"
|
2086
|
-
f"
|
2263
|
+
f" {min_dt_col_name_da} AS {sql_item_name('min_dt', self.flavor)},\n"
|
2264
|
+
f" {max_dt_col_name_da} AS {sql_item_name('max_dt', self.flavor)}\n"
|
2087
2265
|
f"FROM {temp_table_names['new' if not upsert else 'update']}\n"
|
2088
2266
|
f"WHERE {dt_col_name} IS NOT NULL"
|
2089
2267
|
],
|
@@ -2350,6 +2528,7 @@ def sync_pipe_inplace(
|
|
2350
2528
|
patch_schema=internal_schema,
|
2351
2529
|
datetime_col=pipe.columns.get('datetime', None),
|
2352
2530
|
flavor=self.flavor,
|
2531
|
+
null_indices=pipe.null_indices,
|
2353
2532
|
debug=debug,
|
2354
2533
|
)
|
2355
2534
|
if on_cols else []
|
@@ -2643,7 +2822,7 @@ def get_pipe_rowcount(
|
|
2643
2822
|
_cols_names = ['*']
|
2644
2823
|
|
2645
2824
|
src = (
|
2646
|
-
f"SELECT {', '.join(_cols_names)}
|
2825
|
+
f"SELECT {', '.join(_cols_names)}\nFROM {_pipe_name}"
|
2647
2826
|
if not remote
|
2648
2827
|
else get_pipe_query(pipe)
|
2649
2828
|
)
|
@@ -2652,15 +2831,17 @@ def get_pipe_rowcount(
|
|
2652
2831
|
if begin is not None or end is not None:
|
2653
2832
|
query += "\nWHERE"
|
2654
2833
|
if begin is not None:
|
2655
|
-
query +=
|
2656
|
-
|
2657
|
-
|
2834
|
+
query += (
|
2835
|
+
f"\n {dt_name} >= "
|
2836
|
+
+ dateadd_str(self.flavor, datepart='minute', number=0, begin=begin, db_type=dt_db_type)
|
2837
|
+
)
|
2658
2838
|
if end is not None and begin is not None:
|
2659
|
-
query += "AND"
|
2839
|
+
query += "\n AND"
|
2660
2840
|
if end is not None:
|
2661
|
-
query +=
|
2662
|
-
|
2663
|
-
|
2841
|
+
query += (
|
2842
|
+
f"\n {dt_name} < "
|
2843
|
+
+ dateadd_str(self.flavor, datepart='minute', number=0, begin=end, db_type=dt_db_type)
|
2844
|
+
)
|
2664
2845
|
if params is not None:
|
2665
2846
|
from meerschaum.utils.sql import build_where
|
2666
2847
|
existing_cols = pipe.get_columns_types(debug=debug)
|
@@ -2782,13 +2963,21 @@ def clear_pipe(
|
|
2782
2963
|
valid_params = {k: v for k, v in params.items() if k in existing_cols}
|
2783
2964
|
clear_query = (
|
2784
2965
|
f"DELETE FROM {pipe_name}\nWHERE 1 = 1\n"
|
2785
|
-
+ ('
|
2966
|
+
+ ('\n AND ' + build_where(valid_params, self, with_where=False) if valid_params else '')
|
2786
2967
|
+ (
|
2787
|
-
|
2788
|
-
|
2968
|
+
(
|
2969
|
+
f'\n AND {dt_name} >= '
|
2970
|
+
+ dateadd_str(self.flavor, 'day', 0, begin, db_type=dt_db_type)
|
2971
|
+
)
|
2972
|
+
if begin is not None
|
2973
|
+
else ''
|
2789
2974
|
) + (
|
2790
|
-
|
2791
|
-
|
2975
|
+
(
|
2976
|
+
f'\n AND {dt_name} < '
|
2977
|
+
+ dateadd_str(self.flavor, 'day', 0, end, db_type=dt_db_type)
|
2978
|
+
)
|
2979
|
+
if end is not None
|
2980
|
+
else ''
|
2792
2981
|
)
|
2793
2982
|
)
|
2794
2983
|
success = self.exec(clear_query, silent=True, debug=debug) is not None
|
@@ -2999,7 +3188,9 @@ def get_add_columns_queries(
|
|
2999
3188
|
col: get_db_type_from_pd_type(
|
3000
3189
|
df_cols_types[col],
|
3001
3190
|
self.flavor
|
3002
|
-
)
|
3191
|
+
)
|
3192
|
+
for col in new_cols
|
3193
|
+
if col and df_cols_types.get(col, None)
|
3003
3194
|
}
|
3004
3195
|
|
3005
3196
|
alter_table_query = "ALTER TABLE " + sql_item_name(
|
@@ -3086,7 +3277,7 @@ def get_alter_columns_queries(
|
|
3086
3277
|
else [
|
3087
3278
|
col
|
3088
3279
|
for col, typ in df.items()
|
3089
|
-
if typ
|
3280
|
+
if typ.startswith('numeric')
|
3090
3281
|
]
|
3091
3282
|
)
|
3092
3283
|
df_cols_types = (
|
@@ -3166,7 +3357,7 @@ def get_alter_columns_queries(
|
|
3166
3357
|
+ f"{edit_msg}"
|
3167
3358
|
)
|
3168
3359
|
else:
|
3169
|
-
numeric_cols.extend([col for col, typ in pipe.dtypes.items() if typ
|
3360
|
+
numeric_cols.extend([col for col, typ in pipe.dtypes.items() if typ.startswith('numeric')])
|
3170
3361
|
|
3171
3362
|
numeric_type = get_db_type_from_pd_type('numeric', self.flavor, as_sqlalchemy=False)
|
3172
3363
|
text_type = get_db_type_from_pd_type('str', self.flavor, as_sqlalchemy=False)
|
@@ -3391,6 +3582,7 @@ def get_to_sql_dtype(
|
|
3391
3582
|
return {
|
3392
3583
|
col: get_db_type_from_pd_type(typ, self.flavor, as_sqlalchemy=True)
|
3393
3584
|
for col, typ in df_dtypes.items()
|
3585
|
+
if col and typ
|
3394
3586
|
}
|
3395
3587
|
|
3396
3588
|
|