meerschaum 2.4.12__py3-none-any.whl → 2.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/_internal/entry.py +1 -0
- meerschaum/actions/bootstrap.py +9 -11
- meerschaum/actions/delete.py +1 -1
- meerschaum/actions/edit.py +40 -14
- meerschaum/actions/sh.py +11 -10
- meerschaum/actions/start.py +58 -2
- meerschaum/actions/sync.py +14 -16
- meerschaum/actions/upgrade.py +8 -6
- meerschaum/config/_version.py +1 -1
- meerschaum/connectors/sql/_create_engine.py +8 -3
- meerschaum/connectors/sql/_fetch.py +4 -2
- meerschaum/connectors/sql/_instance.py +3 -3
- meerschaum/connectors/sql/_pipes.py +18 -13
- meerschaum/connectors/sql/_sql.py +59 -50
- meerschaum/core/Pipe/__init__.py +23 -1
- meerschaum/core/Pipe/_attributes.py +96 -14
- meerschaum/plugins/__init__.py +6 -2
- meerschaum/plugins/bootstrap.py +15 -15
- meerschaum/utils/dataframe.py +17 -5
- meerschaum/utils/packages/__init__.py +40 -22
- meerschaum/utils/packages/_packages.py +24 -8
- meerschaum/utils/process.py +18 -8
- meerschaum/utils/schedule.py +9 -5
- meerschaum/utils/venv/__init__.py +35 -24
- meerschaum/utils/warnings.py +7 -7
- {meerschaum-2.4.12.dist-info → meerschaum-2.5.0.dist-info}/METADATA +9 -13
- {meerschaum-2.4.12.dist-info → meerschaum-2.5.0.dist-info}/RECORD +33 -33
- {meerschaum-2.4.12.dist-info → meerschaum-2.5.0.dist-info}/LICENSE +0 -0
- {meerschaum-2.4.12.dist-info → meerschaum-2.5.0.dist-info}/NOTICE +0 -0
- {meerschaum-2.4.12.dist-info → meerschaum-2.5.0.dist-info}/WHEEL +0 -0
- {meerschaum-2.4.12.dist-info → meerschaum-2.5.0.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.4.12.dist-info → meerschaum-2.5.0.dist-info}/top_level.txt +0 -0
- {meerschaum-2.4.12.dist-info → meerschaum-2.5.0.dist-info}/zip-safe +0 -0
meerschaum/_internal/entry.py
CHANGED
meerschaum/actions/bootstrap.py
CHANGED
@@ -19,7 +19,7 @@ def bootstrap(
|
|
19
19
|
) -> SuccessTuple:
|
20
20
|
"""
|
21
21
|
Launch an interactive wizard to bootstrap pipes or connectors.
|
22
|
-
|
22
|
+
|
23
23
|
Example:
|
24
24
|
`bootstrap pipes`
|
25
25
|
|
@@ -462,7 +462,6 @@ def _bootstrap_jobs(
|
|
462
462
|
if not action:
|
463
463
|
action = [prompt("What is the name of the job you'd like to create?")]
|
464
464
|
|
465
|
-
new_jobs = {}
|
466
465
|
for name in action:
|
467
466
|
clear_screen(debug=debug)
|
468
467
|
job = mrsm.Job(name, executor_keys=executor_keys)
|
@@ -473,13 +472,14 @@ def _bootstrap_jobs(
|
|
473
472
|
continue
|
474
473
|
|
475
474
|
info(
|
476
|
-
"
|
475
|
+
f"Editing arguments for job '{name}'.\n"
|
476
|
+
" Press [Esc + Enter] to submit, [CTRL + C] to exit.\n\n"
|
477
477
|
" Tip: join multiple actions with `+`, add pipeline arguments with `:`.\n"
|
478
478
|
" https://meerschaum.io/reference/actions/#chaining-actions\n"
|
479
479
|
)
|
480
480
|
try:
|
481
481
|
new_sysargs_str = prompt(
|
482
|
-
|
482
|
+
"",
|
483
483
|
multiline=True,
|
484
484
|
icon=False,
|
485
485
|
completer=ShellCompleter(),
|
@@ -491,6 +491,7 @@ def _bootstrap_jobs(
|
|
491
491
|
new_sysargs, pipeline_args = split_pipeline_sysargs(new_sysargs)
|
492
492
|
chained_sysargs = split_chained_sysargs(new_sysargs)
|
493
493
|
|
494
|
+
clear_screen(debug=debug)
|
494
495
|
if len(chained_sysargs) > 1:
|
495
496
|
print_options(
|
496
497
|
[
|
@@ -508,6 +509,7 @@ def _bootstrap_jobs(
|
|
508
509
|
if pipeline_args:
|
509
510
|
print('\n' + make_header("Pipeline Arguments:"))
|
510
511
|
print(shlex.join(pipeline_args))
|
512
|
+
print()
|
511
513
|
|
512
514
|
if not yes_no(
|
513
515
|
(
|
@@ -524,15 +526,11 @@ def _bootstrap_jobs(
|
|
524
526
|
if not start_success:
|
525
527
|
return start_success, start_msg
|
526
528
|
|
527
|
-
new_jobs[name] = new_job
|
528
|
-
|
529
|
-
if not new_jobs:
|
530
|
-
return False, "No new jobs were created."
|
531
|
-
|
532
529
|
msg = (
|
533
530
|
"Successfully bootstrapped job"
|
534
|
-
+ ('s' if len(
|
535
|
-
+
|
531
|
+
+ ('s' if len(action) != 1 else '')
|
532
|
+
+ ' '
|
533
|
+
+ items_str(action)
|
536
534
|
+ '.'
|
537
535
|
)
|
538
536
|
return True, msg
|
meerschaum/actions/delete.py
CHANGED
@@ -498,7 +498,7 @@ def _complete_delete_jobs(
|
|
498
498
|
action: Optional[List[str]] = None,
|
499
499
|
executor_keys: Optional[str] = None,
|
500
500
|
line: str = '',
|
501
|
-
_get_job_method:
|
501
|
+
_get_job_method: Union[str, List[str], None] = None,
|
502
502
|
**kw
|
503
503
|
) -> List[str]:
|
504
504
|
from meerschaum._internal.shell.Shell import shell_attrs
|
meerschaum/actions/edit.py
CHANGED
@@ -38,10 +38,17 @@ def _complete_edit(
|
|
38
38
|
"""
|
39
39
|
Override the default Meerschaum `complete_` function.
|
40
40
|
"""
|
41
|
+
from meerschaum.actions.delete import _complete_delete_jobs
|
42
|
+
|
43
|
+
if action is None:
|
44
|
+
action = []
|
45
|
+
|
41
46
|
options = {
|
42
47
|
'config': _complete_edit_config,
|
43
48
|
'plugin': _complete_edit_plugins,
|
44
49
|
'plugins': _complete_edit_plugins,
|
50
|
+
'job': _complete_delete_jobs,
|
51
|
+
'jobs': _complete_delete_jobs,
|
45
52
|
}
|
46
53
|
|
47
54
|
if action is None:
|
@@ -133,7 +140,7 @@ def _edit_pipes(
|
|
133
140
|
|
134
141
|
pipes = get_pipes(debug=debug, as_list=True, **kw)
|
135
142
|
if pipes:
|
136
|
-
print_options(pipes, header=
|
143
|
+
print_options(pipes, header='Pipes to be edited:')
|
137
144
|
else:
|
138
145
|
return False, "No pipes to edit."
|
139
146
|
|
@@ -144,10 +151,10 @@ def _edit_pipes(
|
|
144
151
|
f"Press [Enter] to begin editing the above {len(pipes)} pipe"
|
145
152
|
+ ("s" if len(pipes) != 1 else "")
|
146
153
|
+ " or [CTRL-C] to cancel:",
|
147
|
-
icon
|
154
|
+
icon=False,
|
148
155
|
)
|
149
156
|
except KeyboardInterrupt:
|
150
|
-
return False,
|
157
|
+
return False, "No pipes changed."
|
151
158
|
|
152
159
|
interactive = (not bool(params))
|
153
160
|
success, msg = True, ""
|
@@ -393,41 +400,51 @@ def _edit_jobs(
|
|
393
400
|
)
|
394
401
|
from meerschaum.utils.formatting import make_header, print_options
|
395
402
|
from meerschaum.utils.warnings import info
|
396
|
-
from meerschaum.
|
403
|
+
from meerschaum._internal.shell.ShellCompleter import ShellCompleter
|
404
|
+
from meerschaum.utils.misc import items_str
|
405
|
+
from meerschaum.utils.formatting._shell import clear_screen
|
406
|
+
|
397
407
|
jobs = get_filtered_jobs(executor_keys, action, debug=debug)
|
398
408
|
if not jobs:
|
399
409
|
return False, "No jobs to edit."
|
400
410
|
|
401
|
-
|
402
|
-
"Press [Esc + Enter] to submit, [CTRL + C] to exit.\n"
|
403
|
-
" Tip: join multiple actions with `+`, add pipeline arguments with `:`.\n"
|
404
|
-
" https://meerschaum.io/reference/actions/#chaining-actions\n"
|
405
|
-
)
|
406
|
-
|
411
|
+
num_edited = 0
|
407
412
|
for name, job in jobs.items():
|
408
413
|
sysargs_str = shlex.join(job.sysargs)
|
414
|
+
clear_screen(debug=debug)
|
415
|
+
info(
|
416
|
+
f"Editing arguments for job '{name}'.\n"
|
417
|
+
" Press [Esc + Enter] to submit, [CTRL + C] to exit.\n\n"
|
418
|
+
" Tip: join actions with `+`, manage pipeline with `:`.\n"
|
419
|
+
" https://meerschaum.io/reference/actions/#chaining-actions\n"
|
420
|
+
)
|
409
421
|
|
410
422
|
try:
|
411
423
|
new_sysargs_str = prompt(
|
412
|
-
|
413
|
-
default_editable=sysargs_str.lstrip().rstrip(),
|
424
|
+
"",
|
425
|
+
default_editable=sysargs_str.lstrip().rstrip().replace(' + ', '\n+ '),
|
414
426
|
multiline=True,
|
415
427
|
icon=False,
|
428
|
+
completer=ShellCompleter(),
|
416
429
|
)
|
417
430
|
except KeyboardInterrupt:
|
418
431
|
return True, "Nothing was changed."
|
419
432
|
|
433
|
+
if new_sysargs_str.strip() == sysargs_str.strip():
|
434
|
+
continue
|
435
|
+
|
420
436
|
new_sysargs = shlex.split(new_sysargs_str)
|
421
437
|
new_sysargs, pipeline_args = split_pipeline_sysargs(new_sysargs)
|
422
438
|
chained_sysargs = split_chained_sysargs(new_sysargs)
|
423
439
|
|
440
|
+
clear_screen(debug=debug)
|
424
441
|
if len(chained_sysargs) > 1:
|
425
442
|
print_options(
|
426
443
|
[
|
427
444
|
shlex.join(step_sysargs)
|
428
445
|
for step_sysargs in chained_sysargs
|
429
446
|
],
|
430
|
-
header=f"
|
447
|
+
header=f"\nSteps in Job '{name}':",
|
431
448
|
number_options=True,
|
432
449
|
**kwargs
|
433
450
|
)
|
@@ -438,6 +455,7 @@ def _edit_jobs(
|
|
438
455
|
if pipeline_args:
|
439
456
|
print('\n' + make_header("Pipeline Arguments:"))
|
440
457
|
print(shlex.join(pipeline_args))
|
458
|
+
print()
|
441
459
|
|
442
460
|
if not yes_no(
|
443
461
|
(
|
@@ -457,8 +475,16 @@ def _edit_jobs(
|
|
457
475
|
start_success, start_msg = new_job.start()
|
458
476
|
if not start_success:
|
459
477
|
return start_success, start_msg
|
478
|
+
num_edited += 1
|
460
479
|
|
461
|
-
|
480
|
+
msg = (
|
481
|
+
"Successfully edit job"
|
482
|
+
+ ('s' if len(jobs) != 1 else '')
|
483
|
+
+ ' '
|
484
|
+
+ items_str(list(jobs.keys()))
|
485
|
+
+ '.'
|
486
|
+
) if num_edited > 0 else "Nothing was edited."
|
487
|
+
return True, msg
|
462
488
|
|
463
489
|
|
464
490
|
### NOTE: This must be the final statement of the module.
|
meerschaum/actions/sh.py
CHANGED
@@ -9,14 +9,15 @@ NOTE: This action may be a huge security vulnerability
|
|
9
9
|
|
10
10
|
from meerschaum.utils.typing import SuccessTuple, List, Any, Optional
|
11
11
|
|
12
|
+
|
12
13
|
def sh(
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
14
|
+
action: Optional[List[str]] = None,
|
15
|
+
sub_args: Optional[List[str]] = None,
|
16
|
+
filtered_sysargs: Optional[List[str]] = None,
|
17
|
+
use_bash: bool = True,
|
18
|
+
debug: bool = False,
|
19
|
+
**kw: Any
|
20
|
+
) -> SuccessTuple:
|
20
21
|
"""
|
21
22
|
Execute system commands.
|
22
23
|
"""
|
@@ -50,7 +51,7 @@ def sh(
|
|
50
51
|
if len(action) != 0:
|
51
52
|
try:
|
52
53
|
command_list += ["-c", shlex.join(cmd_list)]
|
53
|
-
except Exception
|
54
|
+
except Exception:
|
54
55
|
command_list += ["-c", ' '.join(cmd_list)]
|
55
56
|
else:
|
56
57
|
if len(action) == 0:
|
@@ -64,8 +65,8 @@ def sh(
|
|
64
65
|
try:
|
65
66
|
process = subprocess.Popen(
|
66
67
|
command_list,
|
67
|
-
shell
|
68
|
-
env
|
68
|
+
shell=False,
|
69
|
+
env=os.environ,
|
69
70
|
)
|
70
71
|
exit_code = process.wait()
|
71
72
|
except FileNotFoundError:
|
meerschaum/actions/start.py
CHANGED
@@ -541,6 +541,7 @@ def _start_pipeline(
|
|
541
541
|
action: Optional[List[str]] = None,
|
542
542
|
loop: bool = False,
|
543
543
|
min_seconds: Union[float, int, None] = 1.0,
|
544
|
+
timeout_seconds: Optional[int] = None,
|
544
545
|
params: Optional[Dict[str, Any]] = None,
|
545
546
|
**kwargs
|
546
547
|
) -> SuccessTuple:
|
@@ -557,10 +558,39 @@ def _start_pipeline(
|
|
557
558
|
`show version + show arguments :: --loop`
|
558
559
|
|
559
560
|
"""
|
561
|
+
import json
|
560
562
|
import time
|
563
|
+
import sys
|
561
564
|
from meerschaum._internal.entry import entry
|
562
565
|
from meerschaum.utils.warnings import info, warn
|
563
566
|
from meerschaum.utils.misc import is_int
|
567
|
+
from meerschaum.utils.venv import venv_exec
|
568
|
+
from meerschaum.utils.process import poll_process
|
569
|
+
fence_begin, fence_end = '<MRSM_RESULT>', '</MRSM_RESULT>'
|
570
|
+
|
571
|
+
success, msg = False, "Did not run pipeline."
|
572
|
+
def write_line(line):
|
573
|
+
nonlocal success, msg
|
574
|
+
decoded = line.decode('utf-8')
|
575
|
+
begin_index, end_index = decoded.find(fence_begin), decoded.find(fence_end)
|
576
|
+
|
577
|
+
### Found the beginning of the return value.
|
578
|
+
### Don't write the parsed success tuple message.
|
579
|
+
if begin_index >= 0:
|
580
|
+
success, msg = tuple(json.loads(
|
581
|
+
decoded[begin_index + len(fence_begin):end_index]
|
582
|
+
))
|
583
|
+
return
|
584
|
+
|
585
|
+
print(decoded)
|
586
|
+
|
587
|
+
def timeout_handler(*args, **kw):
|
588
|
+
nonlocal success, msg
|
589
|
+
success, msg = False, (
|
590
|
+
f"Failed to execute pipeline within {timeout_seconds} second"
|
591
|
+
+ ('s' if timeout_seconds != 1 else '') + '.'
|
592
|
+
)
|
593
|
+
write_line((fence_begin + json.dumps((success, msg)) + fence_end).encode('utf-8'))
|
564
594
|
|
565
595
|
do_n_times = (
|
566
596
|
int(action[0].lstrip('x'))
|
@@ -578,12 +608,38 @@ def _start_pipeline(
|
|
578
608
|
if min_seconds is None:
|
579
609
|
min_seconds = 1.0
|
580
610
|
|
611
|
+
def do_entry() -> None:
|
612
|
+
nonlocal success, msg
|
613
|
+
if timeout_seconds is None:
|
614
|
+
success, msg = entry(sub_args_line, _patch_args=patch_args)
|
615
|
+
return
|
616
|
+
|
617
|
+
sub_args_line_escaped = sub_args_line.replace("'", "<QUOTE>")
|
618
|
+
patch_args_escaped_str = json.dumps(patch_args).replace("'", "<QUOTE>")
|
619
|
+
src = (
|
620
|
+
"import json\n"
|
621
|
+
"from meerschaum._internal.entry import entry\n\n"
|
622
|
+
f"sub_args_line = '{sub_args_line_escaped}'.replace(\"<QUOTE>\", \"'\")\n"
|
623
|
+
f"patch_args = json.loads('{patch_args_escaped_str}'.replace('<QUOTE>', \"'\"))\n"
|
624
|
+
"success, msg = entry(sub_args_line, _patch_args=patch_args)\n"
|
625
|
+
f"print('{fence_begin}' + json.dumps((success, msg)) + '{fence_end}')"
|
626
|
+
)
|
627
|
+
proc = venv_exec(src, venv=None, as_proc=True)
|
628
|
+
poll_process(
|
629
|
+
proc,
|
630
|
+
write_line,
|
631
|
+
timeout_seconds,
|
632
|
+
timeout_handler,
|
633
|
+
)
|
634
|
+
|
581
635
|
ran_n_times = 0
|
582
|
-
success, msg = False, "Did not run pipeline."
|
583
636
|
def run_loop():
|
584
637
|
nonlocal ran_n_times, success, msg
|
585
638
|
while True:
|
586
|
-
|
639
|
+
try:
|
640
|
+
do_entry()
|
641
|
+
except Exception as e:
|
642
|
+
warn(e)
|
587
643
|
ran_n_times += 1
|
588
644
|
|
589
645
|
if not loop and do_n_times == 1:
|
meerschaum/actions/sync.py
CHANGED
@@ -129,7 +129,6 @@ def _pipes_lap(
|
|
129
129
|
stack=False,
|
130
130
|
)
|
131
131
|
|
132
|
-
|
133
132
|
def _task_label(count: int):
|
134
133
|
return f"[cyan]Syncing {count} pipe{'s' if count != 1 else ''}..."
|
135
134
|
|
@@ -137,7 +136,6 @@ def _pipes_lap(
|
|
137
136
|
_progress.add_task(_task_label(len(pipes)), start=True, total=len(pipes))
|
138
137
|
) if _progress is not None else None
|
139
138
|
|
140
|
-
|
141
139
|
def worker_fn():
|
142
140
|
while not stop_event.is_set():
|
143
141
|
try:
|
@@ -188,7 +186,7 @@ def _pipes_lap(
|
|
188
186
|
decoded[begin_index + len(fence_begin):end_index]
|
189
187
|
))
|
190
188
|
return
|
191
|
-
|
189
|
+
print(decoded)
|
192
190
|
|
193
191
|
def timeout_handler(p, *args, **kw):
|
194
192
|
success, msg = False, (
|
@@ -227,7 +225,7 @@ def _pipes_lap(
|
|
227
225
|
write_line,
|
228
226
|
timeout_seconds,
|
229
227
|
timeout_handler,
|
230
|
-
(p,)
|
228
|
+
(p,),
|
231
229
|
)
|
232
230
|
return _success_tuple
|
233
231
|
|
@@ -249,18 +247,18 @@ def _pipes_lap(
|
|
249
247
|
|
250
248
|
|
251
249
|
def _sync_pipes(
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
250
|
+
loop: bool = False,
|
251
|
+
min_seconds: int = 1,
|
252
|
+
unblock: bool = False,
|
253
|
+
verify: bool = False,
|
254
|
+
deduplicate: bool = False,
|
255
|
+
bounded: Optional[bool] = None,
|
256
|
+
chunk_interval: Union[timedelta, int, None] = None,
|
257
|
+
shell: bool = False,
|
258
|
+
nopretty: bool = False,
|
259
|
+
debug: bool = False,
|
260
|
+
**kw: Any
|
261
|
+
) -> SuccessTuple:
|
264
262
|
"""
|
265
263
|
Fetch and sync new data for pipes.
|
266
264
|
|
meerschaum/actions/upgrade.py
CHANGED
@@ -9,16 +9,17 @@ Upgrade your current Meerschaum environment
|
|
9
9
|
from __future__ import annotations
|
10
10
|
from meerschaum.utils.typing import SuccessTuple, Any, List, Optional, Union
|
11
11
|
|
12
|
+
|
12
13
|
def upgrade(
|
13
14
|
action: Optional[List[str]] = None,
|
14
15
|
**kw: Any
|
15
16
|
) -> SuccessTuple:
|
16
17
|
"""
|
17
18
|
Upgrade Meerschaum, plugins, or packages.
|
18
|
-
|
19
|
+
|
19
20
|
Command:
|
20
21
|
`upgrade {option}`
|
21
|
-
|
22
|
+
|
22
23
|
Example:
|
23
24
|
`upgrade meerschaum`
|
24
25
|
"""
|
@@ -77,7 +78,7 @@ def _upgrade_meerschaum(
|
|
77
78
|
if force:
|
78
79
|
answer = True
|
79
80
|
else:
|
80
|
-
answer = yes_no(
|
81
|
+
answer = yes_no("Take down the stack?", default='n', yes=yes, noask=noask)
|
81
82
|
|
82
83
|
if answer:
|
83
84
|
if debug:
|
@@ -95,7 +96,7 @@ def _upgrade_meerschaum(
|
|
95
96
|
if debug:
|
96
97
|
dprint('Upgrade meerschaum with dependencies: \"' + f'{dependencies}' + '\"')
|
97
98
|
if not pip_install(install_name, venv=None, debug=debug):
|
98
|
-
return False,
|
99
|
+
return False, "Failed to upgrade Meerschaum via pip."
|
99
100
|
|
100
101
|
if debug:
|
101
102
|
dprint("Pulling new Docker images...")
|
@@ -108,6 +109,7 @@ def _upgrade_meerschaum(
|
|
108
109
|
class NoVenv:
|
109
110
|
pass
|
110
111
|
|
112
|
+
|
111
113
|
def _upgrade_packages(
|
112
114
|
action: Optional[List[str]] = None,
|
113
115
|
venv: Union[str, None, NoVenv] = NoVenv,
|
@@ -120,7 +122,7 @@ def _upgrade_packages(
|
|
120
122
|
"""
|
121
123
|
Upgrade and install dependencies.
|
122
124
|
If provided, upgrade only a dependency group, otherwise default to `full`.
|
123
|
-
|
125
|
+
|
124
126
|
Examples:
|
125
127
|
upgrade packages
|
126
128
|
upgrade packages full
|
@@ -159,7 +161,7 @@ def _upgrade_packages(
|
|
159
161
|
to_install = [
|
160
162
|
install_name
|
161
163
|
for install_name in to_install
|
162
|
-
if install_name not in prereleases_to_install
|
164
|
+
if (install_name not in prereleases_to_install) or group == '_internal'
|
163
165
|
]
|
164
166
|
|
165
167
|
success, msg = False, f"Nothing installed."
|
meerschaum/config/_version.py
CHANGED
@@ -63,6 +63,8 @@ flavor_configs = {
|
|
63
63
|
'fast_executemany': True,
|
64
64
|
'isolation_level': 'AUTOCOMMIT',
|
65
65
|
'use_setinputsizes': False,
|
66
|
+
'pool_pre_ping': True,
|
67
|
+
'ignore_no_transaction_on_rollback': True,
|
66
68
|
},
|
67
69
|
'omit_create_engine': {'method',},
|
68
70
|
'to_sql': {
|
@@ -189,15 +191,18 @@ def create_engine(
|
|
189
191
|
### Install and patch required drivers.
|
190
192
|
if self.flavor in install_flavor_drivers:
|
191
193
|
attempt_import(*install_flavor_drivers[self.flavor], debug=debug, lazy=False, warn=False)
|
194
|
+
if self.flavor == 'mssql':
|
195
|
+
pyodbc = attempt_import('pyodbc', debug=debug, lazy=False, warn=False)
|
196
|
+
pyodbc.pooling = False
|
192
197
|
if self.flavor in require_patching_flavors:
|
193
198
|
from meerschaum.utils.packages import determine_version, _monkey_patch_get_distribution
|
194
199
|
import pathlib
|
195
200
|
for install_name, import_name in require_patching_flavors[self.flavor]:
|
196
201
|
pkg = attempt_import(
|
197
202
|
import_name,
|
198
|
-
debug
|
199
|
-
lazy
|
200
|
-
warn
|
203
|
+
debug=debug,
|
204
|
+
lazy=False,
|
205
|
+
warn=False
|
201
206
|
)
|
202
207
|
_monkey_patch_get_distribution(
|
203
208
|
install_name, determine_version(pathlib.Path(pkg.__file__), venv='mrsm')
|
@@ -31,7 +31,7 @@ def fetch(
|
|
31
31
|
----------
|
32
32
|
pipe: mrsm.Pipe
|
33
33
|
The pipe object which contains the `fetch` metadata.
|
34
|
-
|
34
|
+
|
35
35
|
- pipe.columns['datetime']: str
|
36
36
|
- Name of the datetime column for the remote table.
|
37
37
|
- pipe.parameters['fetch']: Dict[str, Any]
|
@@ -196,7 +196,7 @@ def get_pipe_metadef(
|
|
196
196
|
dateadd_str(
|
197
197
|
flavor=self.flavor,
|
198
198
|
datepart='minute',
|
199
|
-
number=((-1 * btm) if apply_backtrack else 0),
|
199
|
+
number=((-1 * btm) if apply_backtrack else 0),
|
200
200
|
begin=begin,
|
201
201
|
)
|
202
202
|
if begin
|
@@ -309,6 +309,8 @@ def _simple_fetch_query(
|
|
309
309
|
"""Build a fetch query from a pipe's definition."""
|
310
310
|
from meerschaum.utils.sql import format_cte_subquery
|
311
311
|
definition = get_pipe_query(pipe)
|
312
|
+
if definition is None:
|
313
|
+
raise ValueError(f"No SQL definition could be found for {pipe}.")
|
312
314
|
return format_cte_subquery(definition, flavor, 'definition')
|
313
315
|
|
314
316
|
|
@@ -88,9 +88,9 @@ def _drop_temporary_tables(self, debug: bool = False) -> SuccessTuple:
|
|
88
88
|
from meerschaum.connectors.sql.tables import get_tables
|
89
89
|
sqlalchemy = mrsm.attempt_import('sqlalchemy')
|
90
90
|
temp_tables_table = get_tables(
|
91
|
-
mrsm_instance
|
92
|
-
create
|
93
|
-
debug
|
91
|
+
mrsm_instance=self,
|
92
|
+
create=False,
|
93
|
+
debug=debug,
|
94
94
|
)['temp_tables']
|
95
95
|
query = (
|
96
96
|
sqlalchemy.select(temp_tables_table.c.table)
|
@@ -384,7 +384,7 @@ def get_create_index_queries(
|
|
384
384
|
|
385
385
|
Returns
|
386
386
|
-------
|
387
|
-
A dictionary of
|
387
|
+
A dictionary of index names mapping to lists of queries.
|
388
388
|
"""
|
389
389
|
### NOTE: Due to recent breaking changes in DuckDB, indices don't behave properly.
|
390
390
|
if self.flavor == 'duckdb':
|
@@ -400,7 +400,8 @@ def get_create_index_queries(
|
|
400
400
|
index_queries = {}
|
401
401
|
|
402
402
|
upsert = pipe.parameters.get('upsert', False) and (self.flavor + '-upsert') in update_queries
|
403
|
-
|
403
|
+
index_names = pipe.get_indices()
|
404
|
+
indices = pipe.indices
|
404
405
|
|
405
406
|
_datetime = pipe.get_columns('datetime', error=False)
|
406
407
|
_datetime_type = pipe.dtypes.get(_datetime, 'datetime64[ns]')
|
@@ -409,8 +410,8 @@ def get_create_index_queries(
|
|
409
410
|
if _datetime is not None else None
|
410
411
|
)
|
411
412
|
_datetime_index_name = (
|
412
|
-
sql_item_name(
|
413
|
-
if
|
413
|
+
sql_item_name(index_names['datetime'], self.flavor, None)
|
414
|
+
if index_names.get('datetime', None)
|
414
415
|
else None
|
415
416
|
)
|
416
417
|
_id = pipe.get_columns('id', error=False)
|
@@ -421,8 +422,8 @@ def get_create_index_queries(
|
|
421
422
|
)
|
422
423
|
|
423
424
|
_id_index_name = (
|
424
|
-
sql_item_name(
|
425
|
-
if
|
425
|
+
sql_item_name(index_names['id'], self.flavor, None)
|
426
|
+
if index_names.get('id', None)
|
426
427
|
else None
|
427
428
|
)
|
428
429
|
_pipe_name = sql_item_name(pipe.target, self.flavor, self.get_pipe_schema(pipe))
|
@@ -491,18 +492,22 @@ def get_create_index_queries(
|
|
491
492
|
if id_query is not None:
|
492
493
|
index_queries[_id] = id_query if isinstance(id_query, list) else [id_query]
|
493
494
|
|
494
|
-
|
495
495
|
### Create indices for other labels in `pipe.columns`.
|
496
|
-
|
496
|
+
other_index_names = {
|
497
497
|
ix_key: ix_unquoted
|
498
|
-
for ix_key, ix_unquoted in
|
498
|
+
for ix_key, ix_unquoted in index_names.items()
|
499
499
|
if ix_key not in ('datetime', 'id')
|
500
500
|
}
|
501
|
-
for ix_key, ix_unquoted in
|
501
|
+
for ix_key, ix_unquoted in other_index_names.items():
|
502
502
|
ix_name = sql_item_name(ix_unquoted, self.flavor, None)
|
503
|
-
|
504
|
-
|
505
|
-
|
503
|
+
cols = indices[ix_key]
|
504
|
+
if not isinstance(cols, (list, tuple)):
|
505
|
+
cols = [cols]
|
506
|
+
cols_names = [sql_item_name(col, self.flavor, None) for col in cols if col]
|
507
|
+
if not cols_names:
|
508
|
+
continue
|
509
|
+
cols_names_str = ", ".join(cols_names)
|
510
|
+
index_queries[ix_key] = [f"CREATE INDEX {ix_name} ON {_pipe_name} ({cols_names_str})"]
|
506
511
|
|
507
512
|
existing_cols_types = pipe.get_columns_types(debug=debug)
|
508
513
|
indices_cols_str = ', '.join(
|