meerschaum 2.7.6__py3-none-any.whl → 2.7.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/actions/drop.py +100 -22
- meerschaum/actions/index.py +71 -0
- meerschaum/actions/register.py +8 -12
- meerschaum/actions/sql.py +1 -1
- meerschaum/api/routes/_pipes.py +18 -0
- meerschaum/api/routes/_plugins.py +1 -1
- meerschaum/api/routes/_users.py +62 -61
- meerschaum/config/_version.py +1 -1
- meerschaum/connectors/api/_pipes.py +20 -0
- meerschaum/connectors/sql/_SQLConnector.py +6 -3
- meerschaum/connectors/sql/_create_engine.py +1 -1
- meerschaum/connectors/sql/_fetch.py +4 -9
- meerschaum/connectors/sql/_instance.py +3 -3
- meerschaum/connectors/sql/_pipes.py +255 -66
- meerschaum/connectors/sql/_plugins.py +11 -16
- meerschaum/connectors/sql/_sql.py +4 -8
- meerschaum/connectors/sql/_uri.py +9 -9
- meerschaum/connectors/sql/_users.py +10 -12
- meerschaum/connectors/sql/tables/__init__.py +13 -14
- meerschaum/core/Pipe/__init__.py +12 -2
- meerschaum/core/Pipe/_attributes.py +32 -38
- meerschaum/core/Pipe/_drop.py +73 -2
- meerschaum/core/Pipe/_index.py +68 -0
- meerschaum/utils/dtypes/sql.py +2 -2
- meerschaum/utils/sql.py +80 -34
- {meerschaum-2.7.6.dist-info → meerschaum-2.7.7.dist-info}/METADATA +14 -2
- {meerschaum-2.7.6.dist-info → meerschaum-2.7.7.dist-info}/RECORD +33 -31
- {meerschaum-2.7.6.dist-info → meerschaum-2.7.7.dist-info}/WHEEL +1 -1
- {meerschaum-2.7.6.dist-info → meerschaum-2.7.7.dist-info}/LICENSE +0 -0
- {meerschaum-2.7.6.dist-info → meerschaum-2.7.7.dist-info}/NOTICE +0 -0
- {meerschaum-2.7.6.dist-info → meerschaum-2.7.7.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.7.6.dist-info → meerschaum-2.7.7.dist-info}/top_level.txt +0 -0
- {meerschaum-2.7.6.dist-info → meerschaum-2.7.7.dist-info}/zip-safe +0 -0
@@ -148,7 +148,6 @@ def get_pipe_metadef(
|
|
148
148
|
from meerschaum.utils.warnings import warn
|
149
149
|
from meerschaum.utils.sql import sql_item_name, dateadd_str, build_where
|
150
150
|
from meerschaum.utils.dtypes.sql import get_db_type_from_pd_type
|
151
|
-
from meerschaum.utils.misc import is_int
|
152
151
|
from meerschaum.config import get_config
|
153
152
|
|
154
153
|
dt_col = pipe.columns.get('datetime', None)
|
@@ -191,7 +190,7 @@ def get_pipe_metadef(
|
|
191
190
|
else begin
|
192
191
|
)
|
193
192
|
|
194
|
-
if begin and end and begin >= end:
|
193
|
+
if begin not in (None, '') and end is not None and begin >= end:
|
195
194
|
begin = None
|
196
195
|
|
197
196
|
if dt_name:
|
@@ -203,7 +202,7 @@ def get_pipe_metadef(
|
|
203
202
|
begin=begin,
|
204
203
|
db_type=db_dt_typ,
|
205
204
|
)
|
206
|
-
if begin
|
205
|
+
if begin not in ('', None)
|
207
206
|
else None
|
208
207
|
)
|
209
208
|
end_da = (
|
@@ -214,7 +213,7 @@ def get_pipe_metadef(
|
|
214
213
|
begin=end,
|
215
214
|
db_type=db_dt_typ,
|
216
215
|
)
|
217
|
-
if end
|
216
|
+
if end is not None
|
218
217
|
else None
|
219
218
|
)
|
220
219
|
|
@@ -228,11 +227,7 @@ def get_pipe_metadef(
|
|
228
227
|
|
229
228
|
has_where = 'where' in meta_def.lower()[meta_def.lower().rfind('definition'):]
|
230
229
|
if dt_name and (begin_da or end_da):
|
231
|
-
definition_dt_name =
|
232
|
-
dateadd_str(self.flavor, 'minute', 0, f"{definition_name}.{dt_name}", db_type=db_dt_typ)
|
233
|
-
if not is_int((begin_da or end_da))
|
234
|
-
else f"{definition_name}.{dt_name}"
|
235
|
-
)
|
230
|
+
definition_dt_name = f"{definition_name}.{dt_name}"
|
236
231
|
meta_def += "\n" + ("AND" if has_where else "WHERE") + " "
|
237
232
|
has_where = True
|
238
233
|
if begin_da:
|
@@ -25,7 +25,7 @@ def _log_temporary_tables_creation(
|
|
25
25
|
"""
|
26
26
|
from meerschaum.utils.misc import items_str
|
27
27
|
from meerschaum.connectors.sql.tables import get_tables
|
28
|
-
sqlalchemy = mrsm.attempt_import('sqlalchemy')
|
28
|
+
sqlalchemy = mrsm.attempt_import('sqlalchemy', lazy=False)
|
29
29
|
temp_tables_table = get_tables(
|
30
30
|
mrsm_instance=self,
|
31
31
|
create=create,
|
@@ -86,7 +86,7 @@ def _drop_temporary_tables(self, debug: bool = False) -> SuccessTuple:
|
|
86
86
|
"""
|
87
87
|
from meerschaum.utils.misc import items_str
|
88
88
|
from meerschaum.connectors.sql.tables import get_tables
|
89
|
-
sqlalchemy = mrsm.attempt_import('sqlalchemy')
|
89
|
+
sqlalchemy = mrsm.attempt_import('sqlalchemy', lazy=False)
|
90
90
|
temp_tables_table = get_tables(
|
91
91
|
mrsm_instance=self,
|
92
92
|
create=False,
|
@@ -150,7 +150,7 @@ def _drop_old_temporary_tables(
|
|
150
150
|
"""
|
151
151
|
from meerschaum.config import get_config
|
152
152
|
from meerschaum.connectors.sql.tables import get_tables
|
153
|
-
sqlalchemy = mrsm.attempt_import('sqlalchemy')
|
153
|
+
sqlalchemy = mrsm.attempt_import('sqlalchemy', lazy=False)
|
154
154
|
temp_tables_table = get_tables(mrsm_instance=self, create=False, debug=debug)['temp_tables']
|
155
155
|
last_check = getattr(self, '_stale_temporary_tables_check_timestamp', 0)
|
156
156
|
now_ts = time.perf_counter()
|
@@ -55,7 +55,7 @@ def register_pipe(
|
|
55
55
|
parameters = {}
|
56
56
|
|
57
57
|
import json
|
58
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
58
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
59
59
|
values = {
|
60
60
|
'connector_keys' : pipe.connector_keys,
|
61
61
|
'metric_key' : pipe.metric_key,
|
@@ -118,7 +118,7 @@ def edit_pipe(
|
|
118
118
|
pipes_tbl = get_tables(mrsm_instance=self, create=(not pipe.temporary), debug=debug)['pipes']
|
119
119
|
|
120
120
|
import json
|
121
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
121
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
122
122
|
|
123
123
|
values = {
|
124
124
|
'parameters': (
|
@@ -176,7 +176,10 @@ def fetch_pipes_keys(
|
|
176
176
|
from meerschaum.config.static import STATIC_CONFIG
|
177
177
|
import json
|
178
178
|
from copy import deepcopy
|
179
|
-
sqlalchemy, sqlalchemy_sql_functions = attempt_import(
|
179
|
+
sqlalchemy, sqlalchemy_sql_functions = attempt_import(
|
180
|
+
'sqlalchemy',
|
181
|
+
'sqlalchemy.sql.functions', lazy=False,
|
182
|
+
)
|
180
183
|
coalesce = sqlalchemy_sql_functions.coalesce
|
181
184
|
|
182
185
|
if connector_keys is None:
|
@@ -246,7 +249,7 @@ def fetch_pipes_keys(
|
|
246
249
|
|
247
250
|
q = sqlalchemy.select(*select_cols).where(sqlalchemy.and_(True, *_where))
|
248
251
|
for c, vals in cols.items():
|
249
|
-
if not isinstance(vals, (list, tuple)) or not vals or not
|
252
|
+
if not isinstance(vals, (list, tuple)) or not vals or c not in pipes_tbl.c:
|
250
253
|
continue
|
251
254
|
_in_vals, _ex_vals = separate_negation_values(vals)
|
252
255
|
q = q.where(coalesce(pipes_tbl.c[c], 'None').in_(_in_vals)) if _in_vals else q
|
@@ -306,9 +309,28 @@ def fetch_pipes_keys(
|
|
306
309
|
return [(row[0], row[1], row[2]) for row in rows]
|
307
310
|
|
308
311
|
|
312
|
+
def create_pipe_indices(
|
313
|
+
self,
|
314
|
+
pipe: mrsm.Pipe,
|
315
|
+
columns: Optional[List[str]] = None,
|
316
|
+
debug: bool = False,
|
317
|
+
) -> SuccessTuple:
|
318
|
+
"""
|
319
|
+
Create a pipe's indices.
|
320
|
+
"""
|
321
|
+
success = self.create_indices(pipe, columns=columns, debug=debug)
|
322
|
+
msg = (
|
323
|
+
"Success"
|
324
|
+
if success
|
325
|
+
else f"Failed to create indices for {pipe}."
|
326
|
+
)
|
327
|
+
return success, msg
|
328
|
+
|
329
|
+
|
309
330
|
def create_indices(
|
310
331
|
self,
|
311
332
|
pipe: mrsm.Pipe,
|
333
|
+
columns: Optional[List[str]] = None,
|
312
334
|
indices: Optional[List[str]] = None,
|
313
335
|
debug: bool = False
|
314
336
|
) -> bool:
|
@@ -318,29 +340,51 @@ def create_indices(
|
|
318
340
|
from meerschaum.utils.debug import dprint
|
319
341
|
if debug:
|
320
342
|
dprint(f"Creating indices for {pipe}...")
|
343
|
+
|
321
344
|
if not pipe.indices:
|
322
345
|
warn(f"{pipe} has no index columns; skipping index creation.", stack=False)
|
323
346
|
return True
|
324
347
|
|
348
|
+
cols_to_include = set((columns or []) + (indices or [])) or None
|
349
|
+
|
325
350
|
_ = pipe.__dict__.pop('_columns_indices', None)
|
326
351
|
ix_queries = {
|
327
|
-
|
328
|
-
for
|
329
|
-
if
|
352
|
+
col: queries
|
353
|
+
for col, queries in self.get_create_index_queries(pipe, debug=debug).items()
|
354
|
+
if cols_to_include is None or col in cols_to_include
|
330
355
|
}
|
331
356
|
success = True
|
332
|
-
for
|
357
|
+
for col, queries in ix_queries.items():
|
333
358
|
ix_success = all(self.exec_queries(queries, debug=debug, silent=False))
|
334
359
|
success = success and ix_success
|
335
360
|
if not ix_success:
|
336
|
-
warn(f"Failed to create index on column: {
|
361
|
+
warn(f"Failed to create index on column: {col}")
|
337
362
|
|
338
363
|
return success
|
339
364
|
|
340
365
|
|
366
|
+
def drop_pipe_indices(
|
367
|
+
self,
|
368
|
+
pipe: mrsm.Pipe,
|
369
|
+
columns: Optional[List[str]] = None,
|
370
|
+
debug: bool = False,
|
371
|
+
) -> SuccessTuple:
|
372
|
+
"""
|
373
|
+
Drop a pipe's indices.
|
374
|
+
"""
|
375
|
+
success = self.drop_indices(pipe, columns=columns, debug=debug)
|
376
|
+
msg = (
|
377
|
+
"Success"
|
378
|
+
if success
|
379
|
+
else f"Failed to drop indices for {pipe}."
|
380
|
+
)
|
381
|
+
return success, msg
|
382
|
+
|
383
|
+
|
341
384
|
def drop_indices(
|
342
385
|
self,
|
343
386
|
pipe: mrsm.Pipe,
|
387
|
+
columns: Optional[List[str]] = None,
|
344
388
|
indices: Optional[List[str]] = None,
|
345
389
|
debug: bool = False
|
346
390
|
) -> bool:
|
@@ -350,24 +394,81 @@ def drop_indices(
|
|
350
394
|
from meerschaum.utils.debug import dprint
|
351
395
|
if debug:
|
352
396
|
dprint(f"Dropping indices for {pipe}...")
|
353
|
-
|
354
|
-
|
397
|
+
|
398
|
+
if not pipe.indices:
|
399
|
+
warn(f"No indices to drop for {pipe}.", stack=False)
|
355
400
|
return False
|
401
|
+
|
402
|
+
cols_to_include = set((columns or []) + (indices or [])) or None
|
403
|
+
|
356
404
|
ix_queries = {
|
357
|
-
|
358
|
-
for
|
359
|
-
if
|
405
|
+
col: queries
|
406
|
+
for col, queries in self.get_drop_index_queries(pipe, debug=debug).items()
|
407
|
+
if cols_to_include is None or col in cols_to_include
|
360
408
|
}
|
361
409
|
success = True
|
362
|
-
for
|
363
|
-
ix_success = all(self.exec_queries(queries, debug=debug, silent=
|
410
|
+
for col, queries in ix_queries.items():
|
411
|
+
ix_success = all(self.exec_queries(queries, debug=debug, silent=(not debug)))
|
364
412
|
if not ix_success:
|
365
413
|
success = False
|
366
414
|
if debug:
|
367
|
-
dprint(f"Failed to drop index on column: {
|
415
|
+
dprint(f"Failed to drop index on column: {col}")
|
368
416
|
return success
|
369
417
|
|
370
418
|
|
419
|
+
def get_pipe_index_names(self, pipe: mrsm.Pipe) -> Dict[str, str]:
|
420
|
+
"""
|
421
|
+
Return a dictionary mapping index keys to their names on the database.
|
422
|
+
|
423
|
+
Returns
|
424
|
+
-------
|
425
|
+
A dictionary of index keys to column names.
|
426
|
+
"""
|
427
|
+
from meerschaum.utils.sql import DEFAULT_SCHEMA_FLAVORS
|
428
|
+
_parameters = pipe.parameters
|
429
|
+
_index_template = _parameters.get('index_template', "IX_{schema_str}{target}_{column_names}")
|
430
|
+
_schema = self.get_pipe_schema(pipe)
|
431
|
+
if _schema is None:
|
432
|
+
_schema = (
|
433
|
+
DEFAULT_SCHEMA_FLAVORS.get(self.flavor, None)
|
434
|
+
if self.flavor != 'mssql'
|
435
|
+
else None
|
436
|
+
)
|
437
|
+
schema_str = '' if _schema is None else f'{_schema}_'
|
438
|
+
schema_str = ''
|
439
|
+
_indices = pipe.indices
|
440
|
+
_target = pipe.target
|
441
|
+
_column_names = {
|
442
|
+
ix: (
|
443
|
+
'_'.join(cols)
|
444
|
+
if isinstance(cols, (list, tuple))
|
445
|
+
else str(cols)
|
446
|
+
)
|
447
|
+
for ix, cols in _indices.items()
|
448
|
+
if cols
|
449
|
+
}
|
450
|
+
_index_names = {
|
451
|
+
ix: _index_template.format(
|
452
|
+
target=_target,
|
453
|
+
column_names=column_names,
|
454
|
+
connector_keys=pipe.connector_keys,
|
455
|
+
metric_key=pipe.metric_key,
|
456
|
+
location_key=pipe.location_key,
|
457
|
+
schema_str=schema_str,
|
458
|
+
)
|
459
|
+
for ix, column_names in _column_names.items()
|
460
|
+
}
|
461
|
+
### NOTE: Skip any duplicate indices.
|
462
|
+
seen_index_names = {}
|
463
|
+
for ix, index_name in _index_names.items():
|
464
|
+
if index_name in seen_index_names:
|
465
|
+
continue
|
466
|
+
seen_index_names[index_name] = ix
|
467
|
+
return {
|
468
|
+
ix: index_name
|
469
|
+
for index_name, ix in seen_index_names.items()
|
470
|
+
}
|
471
|
+
|
371
472
|
def get_create_index_queries(
|
372
473
|
self,
|
373
474
|
pipe: mrsm.Pipe,
|
@@ -407,7 +508,11 @@ def get_create_index_queries(
|
|
407
508
|
|
408
509
|
upsert = pipe.parameters.get('upsert', False) and (self.flavor + '-upsert') in UPDATE_QUERIES
|
409
510
|
static = pipe.parameters.get('static', False)
|
511
|
+
null_indices = pipe.parameters.get('null_indices', True)
|
410
512
|
index_names = pipe.get_indices()
|
513
|
+
unique_index_name_unquoted = index_names.get('unique', None) or f'IX_{pipe.target}_unique'
|
514
|
+
if upsert:
|
515
|
+
_ = index_names.pop('unique', None)
|
411
516
|
indices = pipe.indices
|
412
517
|
existing_cols_types = pipe.get_columns_types(debug=debug)
|
413
518
|
existing_cols_pd_types = {
|
@@ -420,11 +525,11 @@ def get_create_index_queries(
|
|
420
525
|
existing_clustered_primary_keys = []
|
421
526
|
for col, col_indices in existing_cols_indices.items():
|
422
527
|
for col_ix_doc in col_indices:
|
423
|
-
existing_ix_names.add(col_ix_doc.get('name',
|
528
|
+
existing_ix_names.add(col_ix_doc.get('name', '').lower())
|
424
529
|
if col_ix_doc.get('type', None) == 'PRIMARY KEY':
|
425
|
-
existing_primary_keys.append(col)
|
530
|
+
existing_primary_keys.append(col.lower())
|
426
531
|
if col_ix_doc.get('clustered', True):
|
427
|
-
existing_clustered_primary_keys.append(col)
|
532
|
+
existing_clustered_primary_keys.append(col.lower())
|
428
533
|
|
429
534
|
_datetime = pipe.get_columns('datetime', error=False)
|
430
535
|
_datetime_name = (
|
@@ -456,7 +561,7 @@ def get_create_index_queries(
|
|
456
561
|
)
|
457
562
|
)
|
458
563
|
primary_key_db_type = (
|
459
|
-
get_db_type_from_pd_type(pipe.dtypes.get(primary_key, 'int'), self.flavor)
|
564
|
+
get_db_type_from_pd_type(pipe.dtypes.get(primary_key, 'int') or 'int', self.flavor)
|
460
565
|
if primary_key
|
461
566
|
else None
|
462
567
|
)
|
@@ -471,6 +576,19 @@ def get_create_index_queries(
|
|
471
576
|
if not existing_clustered_primary_keys and _datetime is not None
|
472
577
|
else "NONCLUSTERED"
|
473
578
|
)
|
579
|
+
include_columns_str = "\n ,".join(
|
580
|
+
[
|
581
|
+
sql_item_name(col, flavor=self.flavor) for col in existing_cols_types
|
582
|
+
if col != _datetime
|
583
|
+
]
|
584
|
+
).rstrip(',')
|
585
|
+
include_clause = (
|
586
|
+
(
|
587
|
+
f"\nINCLUDE (\n {include_columns_str}\n)"
|
588
|
+
)
|
589
|
+
if datetime_clustered == 'NONCLUSTERED'
|
590
|
+
else ''
|
591
|
+
)
|
474
592
|
|
475
593
|
_id_index_name = (
|
476
594
|
sql_item_name(index_names['id'], self.flavor, None)
|
@@ -516,7 +634,7 @@ def get_create_index_queries(
|
|
516
634
|
if self.flavor == 'mssql':
|
517
635
|
dt_query = (
|
518
636
|
f"CREATE {datetime_clustered} INDEX {_datetime_index_name} "
|
519
|
-
f"
|
637
|
+
f"\nON {_pipe_name} ({_datetime_name}){include_clause}"
|
520
638
|
)
|
521
639
|
else:
|
522
640
|
dt_query = (
|
@@ -530,7 +648,7 @@ def get_create_index_queries(
|
|
530
648
|
primary_queries = []
|
531
649
|
if (
|
532
650
|
primary_key is not None
|
533
|
-
and primary_key not in existing_primary_keys
|
651
|
+
and primary_key.lower() not in existing_primary_keys
|
534
652
|
and not static
|
535
653
|
):
|
536
654
|
if autoincrement and primary_key not in existing_cols_pd_types:
|
@@ -659,7 +777,10 @@ def get_create_index_queries(
|
|
659
777
|
other_index_names = {
|
660
778
|
ix_key: ix_unquoted
|
661
779
|
for ix_key, ix_unquoted in index_names.items()
|
662
|
-
if
|
780
|
+
if (
|
781
|
+
ix_key not in ('datetime', 'id', 'primary')
|
782
|
+
and ix_unquoted.lower() not in existing_ix_names
|
783
|
+
)
|
663
784
|
}
|
664
785
|
for ix_key, ix_unquoted in other_index_names.items():
|
665
786
|
ix_name = sql_item_name(ix_unquoted, self.flavor, None)
|
@@ -684,24 +805,29 @@ def get_create_index_queries(
|
|
684
805
|
coalesce_indices_cols_str = ', '.join(
|
685
806
|
[
|
686
807
|
(
|
687
|
-
|
688
|
-
|
689
|
-
|
690
|
-
|
691
|
-
|
692
|
-
|
808
|
+
(
|
809
|
+
"COALESCE("
|
810
|
+
+ sql_item_name(ix, self.flavor)
|
811
|
+
+ ", "
|
812
|
+
+ get_null_replacement(existing_cols_types[ix], self.flavor)
|
813
|
+
+ ") "
|
814
|
+
)
|
815
|
+
if ix_key != 'datetime' and null_indices
|
816
|
+
else sql_item_name(ix, self.flavor)
|
817
|
+
)
|
693
818
|
for ix_key, ix in pipe.columns.items()
|
694
819
|
if ix and ix in existing_cols_types
|
695
820
|
]
|
696
821
|
)
|
697
|
-
unique_index_name = sql_item_name(
|
698
|
-
|
822
|
+
unique_index_name = sql_item_name(unique_index_name_unquoted, self.flavor)
|
823
|
+
constraint_name_unquoted = unique_index_name_unquoted.replace('IX_', 'UQ_')
|
824
|
+
constraint_name = sql_item_name(constraint_name_unquoted, self.flavor)
|
699
825
|
add_constraint_query = (
|
700
826
|
f"ALTER TABLE {_pipe_name} ADD CONSTRAINT {constraint_name} UNIQUE ({indices_cols_str})"
|
701
827
|
)
|
702
828
|
unique_index_cols_str = (
|
703
829
|
indices_cols_str
|
704
|
-
if self.flavor not in COALESCE_UNIQUE_INDEX_FLAVORS
|
830
|
+
if self.flavor not in COALESCE_UNIQUE_INDEX_FLAVORS or not null_indices
|
705
831
|
else coalesce_indices_cols_str
|
706
832
|
)
|
707
833
|
create_unique_index_query = (
|
@@ -737,21 +863,33 @@ def get_drop_index_queries(
|
|
737
863
|
return {}
|
738
864
|
if not pipe.exists(debug=debug):
|
739
865
|
return {}
|
866
|
+
|
867
|
+
from collections import defaultdict
|
740
868
|
from meerschaum.utils.sql import (
|
741
869
|
sql_item_name,
|
742
870
|
table_exists,
|
743
871
|
hypertable_queries,
|
744
|
-
|
872
|
+
DROP_INDEX_IF_EXISTS_FLAVORS,
|
745
873
|
)
|
746
|
-
drop_queries =
|
874
|
+
drop_queries = defaultdict(lambda: [])
|
747
875
|
schema = self.get_pipe_schema(pipe)
|
748
|
-
|
876
|
+
index_schema = schema if self.flavor != 'mssql' else None
|
749
877
|
indices = {
|
750
|
-
|
751
|
-
for
|
878
|
+
ix_key: ix
|
879
|
+
for ix_key, ix in pipe.get_indices().items()
|
752
880
|
}
|
753
|
-
|
881
|
+
cols_indices = pipe.get_columns_indices(debug=debug)
|
882
|
+
existing_indices = set()
|
883
|
+
clustered_ix = None
|
884
|
+
for col, ix_metas in cols_indices.items():
|
885
|
+
for ix_meta in ix_metas:
|
886
|
+
ix_name = ix_meta.get('name', None)
|
887
|
+
if ix_meta.get('clustered', False):
|
888
|
+
clustered_ix = ix_name
|
889
|
+
existing_indices.add(ix_name.lower())
|
890
|
+
pipe_name = sql_item_name(pipe.target, self.flavor, schema)
|
754
891
|
pipe_name_no_schema = sql_item_name(pipe.target, self.flavor, None)
|
892
|
+
upsert = pipe.upsert
|
755
893
|
|
756
894
|
if self.flavor not in hypertable_queries:
|
757
895
|
is_hypertable = False
|
@@ -759,7 +897,7 @@ def get_drop_index_queries(
|
|
759
897
|
is_hypertable_query = hypertable_queries[self.flavor].format(table_name=pipe_name)
|
760
898
|
is_hypertable = self.value(is_hypertable_query, silent=True, debug=debug) is not None
|
761
899
|
|
762
|
-
if_exists_str = "IF EXISTS" if self.flavor in
|
900
|
+
if_exists_str = "IF EXISTS " if self.flavor in DROP_INDEX_IF_EXISTS_FLAVORS else ""
|
763
901
|
if is_hypertable:
|
764
902
|
nuke_queries = []
|
765
903
|
temp_table = '_' + pipe.target + '_temp_migration'
|
@@ -769,21 +907,51 @@ def get_drop_index_queries(
|
|
769
907
|
nuke_queries.append(f"DROP TABLE {if_exists_str} {temp_table_name}")
|
770
908
|
nuke_queries += [
|
771
909
|
f"SELECT * INTO {temp_table_name} FROM {pipe_name}",
|
772
|
-
f"DROP TABLE {if_exists_str}
|
910
|
+
f"DROP TABLE {if_exists_str}{pipe_name}",
|
773
911
|
f"ALTER TABLE {temp_table_name} RENAME TO {pipe_name_no_schema}",
|
774
912
|
]
|
775
913
|
nuke_ix_keys = ('datetime', 'id')
|
776
914
|
nuked = False
|
777
915
|
for ix_key in nuke_ix_keys:
|
778
916
|
if ix_key in indices and not nuked:
|
779
|
-
drop_queries[ix_key]
|
917
|
+
drop_queries[ix_key].extend(nuke_queries)
|
780
918
|
nuked = True
|
781
919
|
|
782
|
-
|
783
|
-
ix_key
|
784
|
-
|
785
|
-
if
|
786
|
-
|
920
|
+
for ix_key, ix_unquoted in indices.items():
|
921
|
+
if ix_key in drop_queries:
|
922
|
+
continue
|
923
|
+
if ix_unquoted.lower() not in existing_indices:
|
924
|
+
continue
|
925
|
+
|
926
|
+
if ix_key == 'unique' and upsert and self.flavor not in ('sqlite',) and not is_hypertable:
|
927
|
+
constraint_name_unquoted = ix_unquoted.replace('IX_', 'UQ_')
|
928
|
+
constraint_name = sql_item_name(constraint_name_unquoted, self.flavor)
|
929
|
+
constraint_or_index = (
|
930
|
+
"CONSTRAINT"
|
931
|
+
if self.flavor not in ('mysql', 'mariadb')
|
932
|
+
else 'INDEX'
|
933
|
+
)
|
934
|
+
drop_queries[ix_key].append(
|
935
|
+
f"ALTER TABLE {pipe_name}\n"
|
936
|
+
f"DROP {constraint_or_index} {constraint_name}"
|
937
|
+
)
|
938
|
+
|
939
|
+
query = (
|
940
|
+
(
|
941
|
+
f"ALTER TABLE {pipe_name}\n"
|
942
|
+
if self.flavor in ('mysql', 'mariadb')
|
943
|
+
else ''
|
944
|
+
)
|
945
|
+
+ f"DROP INDEX {if_exists_str}"
|
946
|
+
+ sql_item_name(ix_unquoted, self.flavor, index_schema)
|
947
|
+
)
|
948
|
+
if self.flavor == 'mssql':
|
949
|
+
query += f"\nON {pipe_name}"
|
950
|
+
if ix_unquoted == clustered_ix:
|
951
|
+
query += "\nWITH (ONLINE = ON, MAXDOP = 4)"
|
952
|
+
drop_queries[ix_key].append(query)
|
953
|
+
|
954
|
+
|
787
955
|
return drop_queries
|
788
956
|
|
789
957
|
|
@@ -796,7 +964,7 @@ def delete_pipe(
|
|
796
964
|
Delete a Pipe's registration.
|
797
965
|
"""
|
798
966
|
from meerschaum.utils.packages import attempt_import
|
799
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
967
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
800
968
|
|
801
969
|
if not pipe.id:
|
802
970
|
return False, f"{pipe} is not registered."
|
@@ -1368,7 +1536,7 @@ def create_pipe_table_from_df(
|
|
1368
1536
|
from meerschaum.utils.dtypes.sql import get_db_type_from_pd_type
|
1369
1537
|
primary_key = pipe.columns.get('primary', None)
|
1370
1538
|
primary_key_typ = (
|
1371
|
-
pipe.dtypes.get(primary_key, str(df.dtypes.get(primary_key)))
|
1539
|
+
pipe.dtypes.get(primary_key, str(df.dtypes.get(primary_key, 'int')))
|
1372
1540
|
if primary_key
|
1373
1541
|
else None
|
1374
1542
|
)
|
@@ -1777,6 +1945,7 @@ def sync_pipe(
|
|
1777
1945
|
patch_schema=self.internal_schema,
|
1778
1946
|
datetime_col=(dt_col if dt_col in update_df.columns else None),
|
1779
1947
|
identity_insert=(autoincrement and primary_key in update_df.columns),
|
1948
|
+
null_indices=pipe.null_indices,
|
1780
1949
|
debug=debug,
|
1781
1950
|
)
|
1782
1951
|
update_results = self.exec_queries(
|
@@ -2077,13 +2246,19 @@ def sync_pipe_inplace(
|
|
2077
2246
|
_ = clean_up_temp_tables()
|
2078
2247
|
return True, f"Inserted {new_count}, updated 0 rows."
|
2079
2248
|
|
2080
|
-
|
2249
|
+
min_dt_col_name_da = dateadd_str(
|
2250
|
+
flavor=self.flavor, begin=f"MIN({dt_col_name})", db_type=dt_db_type,
|
2251
|
+
)
|
2252
|
+
max_dt_col_name_da = dateadd_str(
|
2253
|
+
flavor=self.flavor, begin=f"MAX({dt_col_name})", db_type=dt_db_type,
|
2254
|
+
)
|
2255
|
+
|
2081
2256
|
(new_dt_bounds_success, new_dt_bounds_msg), new_dt_bounds_results = session_execute(
|
2082
2257
|
session,
|
2083
2258
|
[
|
2084
2259
|
"SELECT\n"
|
2085
|
-
f"
|
2086
|
-
f"
|
2260
|
+
f" {min_dt_col_name_da} AS {sql_item_name('min_dt', self.flavor)},\n"
|
2261
|
+
f" {max_dt_col_name_da} AS {sql_item_name('max_dt', self.flavor)}\n"
|
2087
2262
|
f"FROM {temp_table_names['new' if not upsert else 'update']}\n"
|
2088
2263
|
f"WHERE {dt_col_name} IS NOT NULL"
|
2089
2264
|
],
|
@@ -2350,6 +2525,7 @@ def sync_pipe_inplace(
|
|
2350
2525
|
patch_schema=internal_schema,
|
2351
2526
|
datetime_col=pipe.columns.get('datetime', None),
|
2352
2527
|
flavor=self.flavor,
|
2528
|
+
null_indices=pipe.null_indices,
|
2353
2529
|
debug=debug,
|
2354
2530
|
)
|
2355
2531
|
if on_cols else []
|
@@ -2643,7 +2819,7 @@ def get_pipe_rowcount(
|
|
2643
2819
|
_cols_names = ['*']
|
2644
2820
|
|
2645
2821
|
src = (
|
2646
|
-
f"SELECT {', '.join(_cols_names)}
|
2822
|
+
f"SELECT {', '.join(_cols_names)}\nFROM {_pipe_name}"
|
2647
2823
|
if not remote
|
2648
2824
|
else get_pipe_query(pipe)
|
2649
2825
|
)
|
@@ -2652,15 +2828,17 @@ def get_pipe_rowcount(
|
|
2652
2828
|
if begin is not None or end is not None:
|
2653
2829
|
query += "\nWHERE"
|
2654
2830
|
if begin is not None:
|
2655
|
-
query +=
|
2656
|
-
|
2657
|
-
|
2831
|
+
query += (
|
2832
|
+
f"\n {dt_name} >= "
|
2833
|
+
+ dateadd_str(self.flavor, datepart='minute', number=0, begin=begin, db_type=dt_db_type)
|
2834
|
+
)
|
2658
2835
|
if end is not None and begin is not None:
|
2659
|
-
query += "AND"
|
2836
|
+
query += "\n AND"
|
2660
2837
|
if end is not None:
|
2661
|
-
query +=
|
2662
|
-
|
2663
|
-
|
2838
|
+
query += (
|
2839
|
+
f"\n {dt_name} < "
|
2840
|
+
+ dateadd_str(self.flavor, datepart='minute', number=0, begin=end, db_type=dt_db_type)
|
2841
|
+
)
|
2664
2842
|
if params is not None:
|
2665
2843
|
from meerschaum.utils.sql import build_where
|
2666
2844
|
existing_cols = pipe.get_columns_types(debug=debug)
|
@@ -2782,13 +2960,21 @@ def clear_pipe(
|
|
2782
2960
|
valid_params = {k: v for k, v in params.items() if k in existing_cols}
|
2783
2961
|
clear_query = (
|
2784
2962
|
f"DELETE FROM {pipe_name}\nWHERE 1 = 1\n"
|
2785
|
-
+ ('
|
2963
|
+
+ ('\n AND ' + build_where(valid_params, self, with_where=False) if valid_params else '')
|
2786
2964
|
+ (
|
2787
|
-
|
2788
|
-
|
2965
|
+
(
|
2966
|
+
f'\n AND {dt_name} >= '
|
2967
|
+
+ dateadd_str(self.flavor, 'day', 0, begin, db_type=dt_db_type)
|
2968
|
+
)
|
2969
|
+
if begin is not None
|
2970
|
+
else ''
|
2789
2971
|
) + (
|
2790
|
-
|
2791
|
-
|
2972
|
+
(
|
2973
|
+
f'\n AND {dt_name} < '
|
2974
|
+
+ dateadd_str(self.flavor, 'day', 0, end, db_type=dt_db_type)
|
2975
|
+
)
|
2976
|
+
if end is not None
|
2977
|
+
else ''
|
2792
2978
|
)
|
2793
2979
|
)
|
2794
2980
|
success = self.exec(clear_query, silent=True, debug=debug) is not None
|
@@ -2999,7 +3185,9 @@ def get_add_columns_queries(
|
|
2999
3185
|
col: get_db_type_from_pd_type(
|
3000
3186
|
df_cols_types[col],
|
3001
3187
|
self.flavor
|
3002
|
-
)
|
3188
|
+
)
|
3189
|
+
for col in new_cols
|
3190
|
+
if col and df_cols_types.get(col, None)
|
3003
3191
|
}
|
3004
3192
|
|
3005
3193
|
alter_table_query = "ALTER TABLE " + sql_item_name(
|
@@ -3391,6 +3579,7 @@ def get_to_sql_dtype(
|
|
3391
3579
|
return {
|
3392
3580
|
col: get_db_type_from_pd_type(typ, self.flavor, as_sqlalchemy=True)
|
3393
3581
|
for col, typ in df_dtypes.items()
|
3582
|
+
if col and typ
|
3394
3583
|
}
|
3395
3584
|
|
3396
3585
|
|