meerschaum 2.4.12__py3-none-any.whl → 2.5.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. meerschaum/_internal/entry.py +1 -0
  2. meerschaum/actions/bootstrap.py +9 -11
  3. meerschaum/actions/delete.py +1 -1
  4. meerschaum/actions/edit.py +40 -14
  5. meerschaum/actions/sh.py +11 -10
  6. meerschaum/actions/start.py +58 -2
  7. meerschaum/actions/sync.py +14 -16
  8. meerschaum/actions/upgrade.py +8 -6
  9. meerschaum/config/_version.py +1 -1
  10. meerschaum/connectors/sql/_create_engine.py +8 -3
  11. meerschaum/connectors/sql/_fetch.py +4 -2
  12. meerschaum/connectors/sql/_instance.py +3 -3
  13. meerschaum/connectors/sql/_pipes.py +18 -13
  14. meerschaum/connectors/sql/_sql.py +59 -50
  15. meerschaum/core/Pipe/__init__.py +23 -1
  16. meerschaum/core/Pipe/_attributes.py +96 -14
  17. meerschaum/plugins/__init__.py +6 -2
  18. meerschaum/plugins/bootstrap.py +15 -15
  19. meerschaum/utils/dataframe.py +17 -5
  20. meerschaum/utils/packages/__init__.py +40 -22
  21. meerschaum/utils/packages/_packages.py +24 -8
  22. meerschaum/utils/process.py +18 -8
  23. meerschaum/utils/schedule.py +9 -5
  24. meerschaum/utils/venv/__init__.py +35 -24
  25. meerschaum/utils/warnings.py +7 -7
  26. {meerschaum-2.4.12.dist-info → meerschaum-2.5.0.dist-info}/METADATA +9 -13
  27. {meerschaum-2.4.12.dist-info → meerschaum-2.5.0.dist-info}/RECORD +33 -33
  28. {meerschaum-2.4.12.dist-info → meerschaum-2.5.0.dist-info}/LICENSE +0 -0
  29. {meerschaum-2.4.12.dist-info → meerschaum-2.5.0.dist-info}/NOTICE +0 -0
  30. {meerschaum-2.4.12.dist-info → meerschaum-2.5.0.dist-info}/WHEEL +0 -0
  31. {meerschaum-2.4.12.dist-info → meerschaum-2.5.0.dist-info}/entry_points.txt +0 -0
  32. {meerschaum-2.4.12.dist-info → meerschaum-2.5.0.dist-info}/top_level.txt +0 -0
  33. {meerschaum-2.4.12.dist-info → meerschaum-2.5.0.dist-info}/zip-safe +0 -0
@@ -43,6 +43,7 @@ if (_STATIC_CONFIG['environment']['systemd_log_path']) in os.environ:
43
43
  if _systemd_stdin_path:
44
44
  sys.stdin = _StdinFile(_systemd_stdin_path)
45
45
 
46
+
46
47
  def entry(
47
48
  sysargs: Optional[List[str]] = None,
48
49
  _patch_args: Optional[Dict[str, Any]] = None,
@@ -19,7 +19,7 @@ def bootstrap(
19
19
  ) -> SuccessTuple:
20
20
  """
21
21
  Launch an interactive wizard to bootstrap pipes or connectors.
22
-
22
+
23
23
  Example:
24
24
  `bootstrap pipes`
25
25
 
@@ -462,7 +462,6 @@ def _bootstrap_jobs(
462
462
  if not action:
463
463
  action = [prompt("What is the name of the job you'd like to create?")]
464
464
 
465
- new_jobs = {}
466
465
  for name in action:
467
466
  clear_screen(debug=debug)
468
467
  job = mrsm.Job(name, executor_keys=executor_keys)
@@ -473,13 +472,14 @@ def _bootstrap_jobs(
473
472
  continue
474
473
 
475
474
  info(
476
- "Press [Esc + Enter] to submit, [CTRL + C] to exit.\n"
475
+ f"Editing arguments for job '{name}'.\n"
476
+ " Press [Esc + Enter] to submit, [CTRL + C] to exit.\n\n"
477
477
  " Tip: join multiple actions with `+`, add pipeline arguments with `:`.\n"
478
478
  " https://meerschaum.io/reference/actions/#chaining-actions\n"
479
479
  )
480
480
  try:
481
481
  new_sysargs_str = prompt(
482
- f"Arguments for job '{name}':",
482
+ "",
483
483
  multiline=True,
484
484
  icon=False,
485
485
  completer=ShellCompleter(),
@@ -491,6 +491,7 @@ def _bootstrap_jobs(
491
491
  new_sysargs, pipeline_args = split_pipeline_sysargs(new_sysargs)
492
492
  chained_sysargs = split_chained_sysargs(new_sysargs)
493
493
 
494
+ clear_screen(debug=debug)
494
495
  if len(chained_sysargs) > 1:
495
496
  print_options(
496
497
  [
@@ -508,6 +509,7 @@ def _bootstrap_jobs(
508
509
  if pipeline_args:
509
510
  print('\n' + make_header("Pipeline Arguments:"))
510
511
  print(shlex.join(pipeline_args))
512
+ print()
511
513
 
512
514
  if not yes_no(
513
515
  (
@@ -524,15 +526,11 @@ def _bootstrap_jobs(
524
526
  if not start_success:
525
527
  return start_success, start_msg
526
528
 
527
- new_jobs[name] = new_job
528
-
529
- if not new_jobs:
530
- return False, "No new jobs were created."
531
-
532
529
  msg = (
533
530
  "Successfully bootstrapped job"
534
- + ('s' if len(new_jobs) != 1 else '')
535
- + items_str(list(new_jobs.keys()))
531
+ + ('s' if len(action) != 1 else '')
532
+ + ' '
533
+ + items_str(action)
536
534
  + '.'
537
535
  )
538
536
  return True, msg
@@ -498,7 +498,7 @@ def _complete_delete_jobs(
498
498
  action: Optional[List[str]] = None,
499
499
  executor_keys: Optional[str] = None,
500
500
  line: str = '',
501
- _get_job_method: Optional[str, List[str]] = None,
501
+ _get_job_method: Union[str, List[str], None] = None,
502
502
  **kw
503
503
  ) -> List[str]:
504
504
  from meerschaum._internal.shell.Shell import shell_attrs
@@ -38,10 +38,17 @@ def _complete_edit(
38
38
  """
39
39
  Override the default Meerschaum `complete_` function.
40
40
  """
41
+ from meerschaum.actions.delete import _complete_delete_jobs
42
+
43
+ if action is None:
44
+ action = []
45
+
41
46
  options = {
42
47
  'config': _complete_edit_config,
43
48
  'plugin': _complete_edit_plugins,
44
49
  'plugins': _complete_edit_plugins,
50
+ 'job': _complete_delete_jobs,
51
+ 'jobs': _complete_delete_jobs,
45
52
  }
46
53
 
47
54
  if action is None:
@@ -133,7 +140,7 @@ def _edit_pipes(
133
140
 
134
141
  pipes = get_pipes(debug=debug, as_list=True, **kw)
135
142
  if pipes:
136
- print_options(pipes, header=f'Pipes to be edited:')
143
+ print_options(pipes, header='Pipes to be edited:')
137
144
  else:
138
145
  return False, "No pipes to edit."
139
146
 
@@ -144,10 +151,10 @@ def _edit_pipes(
144
151
  f"Press [Enter] to begin editing the above {len(pipes)} pipe"
145
152
  + ("s" if len(pipes) != 1 else "")
146
153
  + " or [CTRL-C] to cancel:",
147
- icon = False,
154
+ icon=False,
148
155
  )
149
156
  except KeyboardInterrupt:
150
- return False, f"No pipes changed."
157
+ return False, "No pipes changed."
151
158
 
152
159
  interactive = (not bool(params))
153
160
  success, msg = True, ""
@@ -393,41 +400,51 @@ def _edit_jobs(
393
400
  )
394
401
  from meerschaum.utils.formatting import make_header, print_options
395
402
  from meerschaum.utils.warnings import info
396
- from meerschaum.actions import actions
403
+ from meerschaum._internal.shell.ShellCompleter import ShellCompleter
404
+ from meerschaum.utils.misc import items_str
405
+ from meerschaum.utils.formatting._shell import clear_screen
406
+
397
407
  jobs = get_filtered_jobs(executor_keys, action, debug=debug)
398
408
  if not jobs:
399
409
  return False, "No jobs to edit."
400
410
 
401
- info(
402
- "Press [Esc + Enter] to submit, [CTRL + C] to exit.\n"
403
- " Tip: join multiple actions with `+`, add pipeline arguments with `:`.\n"
404
- " https://meerschaum.io/reference/actions/#chaining-actions\n"
405
- )
406
-
411
+ num_edited = 0
407
412
  for name, job in jobs.items():
408
413
  sysargs_str = shlex.join(job.sysargs)
414
+ clear_screen(debug=debug)
415
+ info(
416
+ f"Editing arguments for job '{name}'.\n"
417
+ " Press [Esc + Enter] to submit, [CTRL + C] to exit.\n\n"
418
+ " Tip: join actions with `+`, manage pipeline with `:`.\n"
419
+ " https://meerschaum.io/reference/actions/#chaining-actions\n"
420
+ )
409
421
 
410
422
  try:
411
423
  new_sysargs_str = prompt(
412
- f"Arguments for job '{name}':",
413
- default_editable=sysargs_str.lstrip().rstrip(),
424
+ "",
425
+ default_editable=sysargs_str.lstrip().rstrip().replace(' + ', '\n+ '),
414
426
  multiline=True,
415
427
  icon=False,
428
+ completer=ShellCompleter(),
416
429
  )
417
430
  except KeyboardInterrupt:
418
431
  return True, "Nothing was changed."
419
432
 
433
+ if new_sysargs_str.strip() == sysargs_str.strip():
434
+ continue
435
+
420
436
  new_sysargs = shlex.split(new_sysargs_str)
421
437
  new_sysargs, pipeline_args = split_pipeline_sysargs(new_sysargs)
422
438
  chained_sysargs = split_chained_sysargs(new_sysargs)
423
439
 
440
+ clear_screen(debug=debug)
424
441
  if len(chained_sysargs) > 1:
425
442
  print_options(
426
443
  [
427
444
  shlex.join(step_sysargs)
428
445
  for step_sysargs in chained_sysargs
429
446
  ],
430
- header=f"Steps in Job '{name}':",
447
+ header=f"\nSteps in Job '{name}':",
431
448
  number_options=True,
432
449
  **kwargs
433
450
  )
@@ -438,6 +455,7 @@ def _edit_jobs(
438
455
  if pipeline_args:
439
456
  print('\n' + make_header("Pipeline Arguments:"))
440
457
  print(shlex.join(pipeline_args))
458
+ print()
441
459
 
442
460
  if not yes_no(
443
461
  (
@@ -457,8 +475,16 @@ def _edit_jobs(
457
475
  start_success, start_msg = new_job.start()
458
476
  if not start_success:
459
477
  return start_success, start_msg
478
+ num_edited += 1
460
479
 
461
- return True, "Success"
480
+ msg = (
481
+ "Successfully edit job"
482
+ + ('s' if len(jobs) != 1 else '')
483
+ + ' '
484
+ + items_str(list(jobs.keys()))
485
+ + '.'
486
+ ) if num_edited > 0 else "Nothing was edited."
487
+ return True, msg
462
488
 
463
489
 
464
490
  ### NOTE: This must be the final statement of the module.
meerschaum/actions/sh.py CHANGED
@@ -9,14 +9,15 @@ NOTE: This action may be a huge security vulnerability
9
9
 
10
10
  from meerschaum.utils.typing import SuccessTuple, List, Any, Optional
11
11
 
12
+
12
13
  def sh(
13
- action: Optional[List[str]] = None,
14
- sub_args: Optional[List[str]] = None,
15
- filtered_sysargs: Optional[List[str]] = None,
16
- use_bash: bool = True,
17
- debug: bool = False,
18
- **kw: Any
19
- ) -> SuccessTuple:
14
+ action: Optional[List[str]] = None,
15
+ sub_args: Optional[List[str]] = None,
16
+ filtered_sysargs: Optional[List[str]] = None,
17
+ use_bash: bool = True,
18
+ debug: bool = False,
19
+ **kw: Any
20
+ ) -> SuccessTuple:
20
21
  """
21
22
  Execute system commands.
22
23
  """
@@ -50,7 +51,7 @@ def sh(
50
51
  if len(action) != 0:
51
52
  try:
52
53
  command_list += ["-c", shlex.join(cmd_list)]
53
- except Exception as e:
54
+ except Exception:
54
55
  command_list += ["-c", ' '.join(cmd_list)]
55
56
  else:
56
57
  if len(action) == 0:
@@ -64,8 +65,8 @@ def sh(
64
65
  try:
65
66
  process = subprocess.Popen(
66
67
  command_list,
67
- shell = False,
68
- env = os.environ,
68
+ shell=False,
69
+ env=os.environ,
69
70
  )
70
71
  exit_code = process.wait()
71
72
  except FileNotFoundError:
@@ -541,6 +541,7 @@ def _start_pipeline(
541
541
  action: Optional[List[str]] = None,
542
542
  loop: bool = False,
543
543
  min_seconds: Union[float, int, None] = 1.0,
544
+ timeout_seconds: Optional[int] = None,
544
545
  params: Optional[Dict[str, Any]] = None,
545
546
  **kwargs
546
547
  ) -> SuccessTuple:
@@ -557,10 +558,39 @@ def _start_pipeline(
557
558
  `show version + show arguments :: --loop`
558
559
 
559
560
  """
561
+ import json
560
562
  import time
563
+ import sys
561
564
  from meerschaum._internal.entry import entry
562
565
  from meerschaum.utils.warnings import info, warn
563
566
  from meerschaum.utils.misc import is_int
567
+ from meerschaum.utils.venv import venv_exec
568
+ from meerschaum.utils.process import poll_process
569
+ fence_begin, fence_end = '<MRSM_RESULT>', '</MRSM_RESULT>'
570
+
571
+ success, msg = False, "Did not run pipeline."
572
+ def write_line(line):
573
+ nonlocal success, msg
574
+ decoded = line.decode('utf-8')
575
+ begin_index, end_index = decoded.find(fence_begin), decoded.find(fence_end)
576
+
577
+ ### Found the beginning of the return value.
578
+ ### Don't write the parsed success tuple message.
579
+ if begin_index >= 0:
580
+ success, msg = tuple(json.loads(
581
+ decoded[begin_index + len(fence_begin):end_index]
582
+ ))
583
+ return
584
+
585
+ print(decoded)
586
+
587
+ def timeout_handler(*args, **kw):
588
+ nonlocal success, msg
589
+ success, msg = False, (
590
+ f"Failed to execute pipeline within {timeout_seconds} second"
591
+ + ('s' if timeout_seconds != 1 else '') + '.'
592
+ )
593
+ write_line((fence_begin + json.dumps((success, msg)) + fence_end).encode('utf-8'))
564
594
 
565
595
  do_n_times = (
566
596
  int(action[0].lstrip('x'))
@@ -578,12 +608,38 @@ def _start_pipeline(
578
608
  if min_seconds is None:
579
609
  min_seconds = 1.0
580
610
 
611
+ def do_entry() -> None:
612
+ nonlocal success, msg
613
+ if timeout_seconds is None:
614
+ success, msg = entry(sub_args_line, _patch_args=patch_args)
615
+ return
616
+
617
+ sub_args_line_escaped = sub_args_line.replace("'", "<QUOTE>")
618
+ patch_args_escaped_str = json.dumps(patch_args).replace("'", "<QUOTE>")
619
+ src = (
620
+ "import json\n"
621
+ "from meerschaum._internal.entry import entry\n\n"
622
+ f"sub_args_line = '{sub_args_line_escaped}'.replace(\"<QUOTE>\", \"'\")\n"
623
+ f"patch_args = json.loads('{patch_args_escaped_str}'.replace('<QUOTE>', \"'\"))\n"
624
+ "success, msg = entry(sub_args_line, _patch_args=patch_args)\n"
625
+ f"print('{fence_begin}' + json.dumps((success, msg)) + '{fence_end}')"
626
+ )
627
+ proc = venv_exec(src, venv=None, as_proc=True)
628
+ poll_process(
629
+ proc,
630
+ write_line,
631
+ timeout_seconds,
632
+ timeout_handler,
633
+ )
634
+
581
635
  ran_n_times = 0
582
- success, msg = False, "Did not run pipeline."
583
636
  def run_loop():
584
637
  nonlocal ran_n_times, success, msg
585
638
  while True:
586
- success, msg = entry(sub_args_line, _patch_args=patch_args)
639
+ try:
640
+ do_entry()
641
+ except Exception as e:
642
+ warn(e)
587
643
  ran_n_times += 1
588
644
 
589
645
  if not loop and do_n_times == 1:
@@ -129,7 +129,6 @@ def _pipes_lap(
129
129
  stack=False,
130
130
  )
131
131
 
132
-
133
132
  def _task_label(count: int):
134
133
  return f"[cyan]Syncing {count} pipe{'s' if count != 1 else ''}..."
135
134
 
@@ -137,7 +136,6 @@ def _pipes_lap(
137
136
  _progress.add_task(_task_label(len(pipes)), start=True, total=len(pipes))
138
137
  ) if _progress is not None else None
139
138
 
140
-
141
139
  def worker_fn():
142
140
  while not stop_event.is_set():
143
141
  try:
@@ -188,7 +186,7 @@ def _pipes_lap(
188
186
  decoded[begin_index + len(fence_begin):end_index]
189
187
  ))
190
188
  return
191
- sys.stdout.buffer.write(line)
189
+ print(decoded)
192
190
 
193
191
  def timeout_handler(p, *args, **kw):
194
192
  success, msg = False, (
@@ -227,7 +225,7 @@ def _pipes_lap(
227
225
  write_line,
228
226
  timeout_seconds,
229
227
  timeout_handler,
230
- (p,)
228
+ (p,),
231
229
  )
232
230
  return _success_tuple
233
231
 
@@ -249,18 +247,18 @@ def _pipes_lap(
249
247
 
250
248
 
251
249
  def _sync_pipes(
252
- loop: bool = False,
253
- min_seconds: int = 1,
254
- unblock: bool = False,
255
- verify: bool = False,
256
- deduplicate: bool = False,
257
- bounded: Optional[bool] = None,
258
- chunk_interval: Union[timedelta, int, None] = None,
259
- shell: bool = False,
260
- nopretty: bool = False,
261
- debug: bool = False,
262
- **kw: Any
263
- ) -> SuccessTuple:
250
+ loop: bool = False,
251
+ min_seconds: int = 1,
252
+ unblock: bool = False,
253
+ verify: bool = False,
254
+ deduplicate: bool = False,
255
+ bounded: Optional[bool] = None,
256
+ chunk_interval: Union[timedelta, int, None] = None,
257
+ shell: bool = False,
258
+ nopretty: bool = False,
259
+ debug: bool = False,
260
+ **kw: Any
261
+ ) -> SuccessTuple:
264
262
  """
265
263
  Fetch and sync new data for pipes.
266
264
 
@@ -9,16 +9,17 @@ Upgrade your current Meerschaum environment
9
9
  from __future__ import annotations
10
10
  from meerschaum.utils.typing import SuccessTuple, Any, List, Optional, Union
11
11
 
12
+
12
13
  def upgrade(
13
14
  action: Optional[List[str]] = None,
14
15
  **kw: Any
15
16
  ) -> SuccessTuple:
16
17
  """
17
18
  Upgrade Meerschaum, plugins, or packages.
18
-
19
+
19
20
  Command:
20
21
  `upgrade {option}`
21
-
22
+
22
23
  Example:
23
24
  `upgrade meerschaum`
24
25
  """
@@ -77,7 +78,7 @@ def _upgrade_meerschaum(
77
78
  if force:
78
79
  answer = True
79
80
  else:
80
- answer = yes_no(f"Take down the stack?", default='y', yes=yes, noask=noask)
81
+ answer = yes_no("Take down the stack?", default='n', yes=yes, noask=noask)
81
82
 
82
83
  if answer:
83
84
  if debug:
@@ -95,7 +96,7 @@ def _upgrade_meerschaum(
95
96
  if debug:
96
97
  dprint('Upgrade meerschaum with dependencies: \"' + f'{dependencies}' + '\"')
97
98
  if not pip_install(install_name, venv=None, debug=debug):
98
- return False, f"Failed to upgrade Meerschaum via pip."
99
+ return False, "Failed to upgrade Meerschaum via pip."
99
100
 
100
101
  if debug:
101
102
  dprint("Pulling new Docker images...")
@@ -108,6 +109,7 @@ def _upgrade_meerschaum(
108
109
  class NoVenv:
109
110
  pass
110
111
 
112
+
111
113
  def _upgrade_packages(
112
114
  action: Optional[List[str]] = None,
113
115
  venv: Union[str, None, NoVenv] = NoVenv,
@@ -120,7 +122,7 @@ def _upgrade_packages(
120
122
  """
121
123
  Upgrade and install dependencies.
122
124
  If provided, upgrade only a dependency group, otherwise default to `full`.
123
-
125
+
124
126
  Examples:
125
127
  upgrade packages
126
128
  upgrade packages full
@@ -159,7 +161,7 @@ def _upgrade_packages(
159
161
  to_install = [
160
162
  install_name
161
163
  for install_name in to_install
162
- if install_name not in prereleases_to_install
164
+ if (install_name not in prereleases_to_install) or group == '_internal'
163
165
  ]
164
166
 
165
167
  success, msg = False, f"Nothing installed."
@@ -2,4 +2,4 @@
2
2
  Specify the Meerschaum release version.
3
3
  """
4
4
 
5
- __version__ = "2.4.12"
5
+ __version__ = "2.5.0"
@@ -63,6 +63,8 @@ flavor_configs = {
63
63
  'fast_executemany': True,
64
64
  'isolation_level': 'AUTOCOMMIT',
65
65
  'use_setinputsizes': False,
66
+ 'pool_pre_ping': True,
67
+ 'ignore_no_transaction_on_rollback': True,
66
68
  },
67
69
  'omit_create_engine': {'method',},
68
70
  'to_sql': {
@@ -189,15 +191,18 @@ def create_engine(
189
191
  ### Install and patch required drivers.
190
192
  if self.flavor in install_flavor_drivers:
191
193
  attempt_import(*install_flavor_drivers[self.flavor], debug=debug, lazy=False, warn=False)
194
+ if self.flavor == 'mssql':
195
+ pyodbc = attempt_import('pyodbc', debug=debug, lazy=False, warn=False)
196
+ pyodbc.pooling = False
192
197
  if self.flavor in require_patching_flavors:
193
198
  from meerschaum.utils.packages import determine_version, _monkey_patch_get_distribution
194
199
  import pathlib
195
200
  for install_name, import_name in require_patching_flavors[self.flavor]:
196
201
  pkg = attempt_import(
197
202
  import_name,
198
- debug = debug,
199
- lazy = False,
200
- warn = False
203
+ debug=debug,
204
+ lazy=False,
205
+ warn=False
201
206
  )
202
207
  _monkey_patch_get_distribution(
203
208
  install_name, determine_version(pathlib.Path(pkg.__file__), venv='mrsm')
@@ -31,7 +31,7 @@ def fetch(
31
31
  ----------
32
32
  pipe: mrsm.Pipe
33
33
  The pipe object which contains the `fetch` metadata.
34
-
34
+
35
35
  - pipe.columns['datetime']: str
36
36
  - Name of the datetime column for the remote table.
37
37
  - pipe.parameters['fetch']: Dict[str, Any]
@@ -196,7 +196,7 @@ def get_pipe_metadef(
196
196
  dateadd_str(
197
197
  flavor=self.flavor,
198
198
  datepart='minute',
199
- number=((-1 * btm) if apply_backtrack else 0),
199
+ number=((-1 * btm) if apply_backtrack else 0),
200
200
  begin=begin,
201
201
  )
202
202
  if begin
@@ -309,6 +309,8 @@ def _simple_fetch_query(
309
309
  """Build a fetch query from a pipe's definition."""
310
310
  from meerschaum.utils.sql import format_cte_subquery
311
311
  definition = get_pipe_query(pipe)
312
+ if definition is None:
313
+ raise ValueError(f"No SQL definition could be found for {pipe}.")
312
314
  return format_cte_subquery(definition, flavor, 'definition')
313
315
 
314
316
 
@@ -88,9 +88,9 @@ def _drop_temporary_tables(self, debug: bool = False) -> SuccessTuple:
88
88
  from meerschaum.connectors.sql.tables import get_tables
89
89
  sqlalchemy = mrsm.attempt_import('sqlalchemy')
90
90
  temp_tables_table = get_tables(
91
- mrsm_instance = self,
92
- create = False,
93
- debug = debug,
91
+ mrsm_instance=self,
92
+ create=False,
93
+ debug=debug,
94
94
  )['temp_tables']
95
95
  query = (
96
96
  sqlalchemy.select(temp_tables_table.c.table)
@@ -384,7 +384,7 @@ def get_create_index_queries(
384
384
 
385
385
  Returns
386
386
  -------
387
- A dictionary of column names mapping to lists of queries.
387
+ A dictionary of index names mapping to lists of queries.
388
388
  """
389
389
  ### NOTE: Due to recent breaking changes in DuckDB, indices don't behave properly.
390
390
  if self.flavor == 'duckdb':
@@ -400,7 +400,8 @@ def get_create_index_queries(
400
400
  index_queries = {}
401
401
 
402
402
  upsert = pipe.parameters.get('upsert', False) and (self.flavor + '-upsert') in update_queries
403
- indices = pipe.get_indices()
403
+ index_names = pipe.get_indices()
404
+ indices = pipe.indices
404
405
 
405
406
  _datetime = pipe.get_columns('datetime', error=False)
406
407
  _datetime_type = pipe.dtypes.get(_datetime, 'datetime64[ns]')
@@ -409,8 +410,8 @@ def get_create_index_queries(
409
410
  if _datetime is not None else None
410
411
  )
411
412
  _datetime_index_name = (
412
- sql_item_name(indices['datetime'], self.flavor, None)
413
- if indices.get('datetime', None)
413
+ sql_item_name(index_names['datetime'], self.flavor, None)
414
+ if index_names.get('datetime', None)
414
415
  else None
415
416
  )
416
417
  _id = pipe.get_columns('id', error=False)
@@ -421,8 +422,8 @@ def get_create_index_queries(
421
422
  )
422
423
 
423
424
  _id_index_name = (
424
- sql_item_name(indices['id'], self.flavor, None)
425
- if indices.get('id', None)
425
+ sql_item_name(index_names['id'], self.flavor, None)
426
+ if index_names.get('id', None)
426
427
  else None
427
428
  )
428
429
  _pipe_name = sql_item_name(pipe.target, self.flavor, self.get_pipe_schema(pipe))
@@ -491,18 +492,22 @@ def get_create_index_queries(
491
492
  if id_query is not None:
492
493
  index_queries[_id] = id_query if isinstance(id_query, list) else [id_query]
493
494
 
494
-
495
495
  ### Create indices for other labels in `pipe.columns`.
496
- other_indices = {
496
+ other_index_names = {
497
497
  ix_key: ix_unquoted
498
- for ix_key, ix_unquoted in pipe.get_indices().items()
498
+ for ix_key, ix_unquoted in index_names.items()
499
499
  if ix_key not in ('datetime', 'id')
500
500
  }
501
- for ix_key, ix_unquoted in other_indices.items():
501
+ for ix_key, ix_unquoted in other_index_names.items():
502
502
  ix_name = sql_item_name(ix_unquoted, self.flavor, None)
503
- col = pipe.columns[ix_key]
504
- col_name = sql_item_name(col, self.flavor, None)
505
- index_queries[col] = [f"CREATE INDEX {ix_name} ON {_pipe_name} ({col_name})"]
503
+ cols = indices[ix_key]
504
+ if not isinstance(cols, (list, tuple)):
505
+ cols = [cols]
506
+ cols_names = [sql_item_name(col, self.flavor, None) for col in cols if col]
507
+ if not cols_names:
508
+ continue
509
+ cols_names_str = ", ".join(cols_names)
510
+ index_queries[ix_key] = [f"CREATE INDEX {ix_name} ON {_pipe_name} ({cols_names_str})"]
506
511
 
507
512
  existing_cols_types = pipe.get_columns_types(debug=debug)
508
513
  indices_cols_str = ', '.join(