meerschaum 2.7.4__py3-none-any.whl → 2.7.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. meerschaum/_internal/shell/Shell.py +4 -6
  2. meerschaum/_internal/shell/ShellCompleter.py +6 -5
  3. meerschaum/actions/clear.py +6 -3
  4. meerschaum/actions/copy.py +33 -27
  5. meerschaum/actions/sql.py +14 -4
  6. meerschaum/actions/sync.py +22 -18
  7. meerschaum/api/dash/pipes.py +2 -3
  8. meerschaum/config/_default.py +11 -0
  9. meerschaum/config/_version.py +1 -1
  10. meerschaum/connectors/api/_misc.py +3 -2
  11. meerschaum/connectors/api/_pipes.py +8 -9
  12. meerschaum/connectors/sql/_SQLConnector.py +1 -0
  13. meerschaum/connectors/sql/_cli.py +18 -12
  14. meerschaum/connectors/sql/_create_engine.py +1 -0
  15. meerschaum/connectors/sql/_pipes.py +51 -14
  16. meerschaum/connectors/sql/_sql.py +109 -16
  17. meerschaum/jobs/_Job.py +1 -0
  18. meerschaum/plugins/__init__.py +7 -3
  19. meerschaum/utils/daemon/Daemon.py +11 -3
  20. meerschaum/utils/daemon/__init__.py +2 -2
  21. meerschaum/utils/misc.py +7 -6
  22. meerschaum/utils/packages/__init__.py +35 -28
  23. meerschaum/utils/packages/_packages.py +1 -1
  24. meerschaum/utils/prompt.py +54 -36
  25. meerschaum/utils/venv/_Venv.py +6 -1
  26. meerschaum/utils/venv/__init__.py +32 -16
  27. {meerschaum-2.7.4.dist-info → meerschaum-2.7.6.dist-info}/METADATA +4 -4
  28. {meerschaum-2.7.4.dist-info → meerschaum-2.7.6.dist-info}/RECORD +34 -34
  29. {meerschaum-2.7.4.dist-info → meerschaum-2.7.6.dist-info}/WHEEL +1 -1
  30. {meerschaum-2.7.4.dist-info → meerschaum-2.7.6.dist-info}/LICENSE +0 -0
  31. {meerschaum-2.7.4.dist-info → meerschaum-2.7.6.dist-info}/NOTICE +0 -0
  32. {meerschaum-2.7.4.dist-info → meerschaum-2.7.6.dist-info}/entry_points.txt +0 -0
  33. {meerschaum-2.7.4.dist-info → meerschaum-2.7.6.dist-info}/top_level.txt +0 -0
  34. {meerschaum-2.7.4.dist-info → meerschaum-2.7.6.dist-info}/zip-safe +0 -0
@@ -742,8 +742,7 @@ class Shell(cmd.Cmd):
742
742
  """
743
743
  from meerschaum import get_connector
744
744
  from meerschaum.connectors.parse import parse_instance_keys
745
- from meerschaum.utils.warnings import warn, info
746
- from meerschaum.utils.misc import remove_ansi
745
+ from meerschaum.utils.warnings import info
747
746
 
748
747
  if action is None:
749
748
  action = []
@@ -829,7 +828,7 @@ class Shell(cmd.Cmd):
829
828
  """
830
829
  from meerschaum import get_connector
831
830
  from meerschaum.connectors.parse import parse_repo_keys
832
- from meerschaum.utils.warnings import warn, info
831
+ from meerschaum.utils.warnings import info
833
832
 
834
833
  if action is None:
835
834
  action = []
@@ -878,7 +877,6 @@ class Shell(cmd.Cmd):
878
877
 
879
878
  Note that executors are API instances.
880
879
  """
881
- from meerschaum import get_connector
882
880
  from meerschaum.connectors.parse import parse_executor_keys
883
881
  from meerschaum.utils.warnings import warn, info
884
882
  from meerschaum.jobs import get_executor_keys_from_context
@@ -894,7 +892,7 @@ class Shell(cmd.Cmd):
894
892
  executor_keys = get_executor_keys_from_context()
895
893
 
896
894
  if executor_keys == 'systemd' and get_executor_keys_from_context() != 'systemd':
897
- warn(f"Cannot execute via `systemd`, falling back to `local`...", stack=False)
895
+ warn("Cannot execute via `systemd`, falling back to `local`...", stack=False)
898
896
  executor_keys = 'local'
899
897
 
900
898
  conn = parse_executor_keys(executor_keys, debug=debug)
@@ -935,7 +933,7 @@ class Shell(cmd.Cmd):
935
933
  if args['action'][0] not in shell_attrs['_actions']:
936
934
  try:
937
935
  print(textwrap.dedent(getattr(self, f"do_{args['action'][0]}").__doc__))
938
- except Exception as e:
936
+ except Exception:
939
937
  print(f"No help on '{args['action'][0]}'.")
940
938
  return ""
941
939
  parse_help(args)
@@ -7,13 +7,15 @@ Implement the prompt_toolkit Completer base class.
7
7
  """
8
8
 
9
9
  from __future__ import annotations
10
- from prompt_toolkit.completion import Completer, Completion
11
- from meerschaum.utils.typing import Optional
12
- from meerschaum.actions import get_shell, get_completer, get_main_action_name, get_action
10
+ from meerschaum.actions import get_shell, get_main_action_name, get_action
13
11
  from meerschaum._internal.arguments import parse_line
14
12
 
15
13
  from meerschaum.utils.packages import attempt_import, ensure_readline
16
- prompt_toolkit = attempt_import('prompt_toolkit', lazy=False, install=True)
14
+
15
+ prompt_toolkit_completion = attempt_import('prompt_toolkit.completion', lazy=False, install=True)
16
+ Completer = prompt_toolkit_completion.Completer
17
+ Completion = prompt_toolkit_completion.Completion
18
+
17
19
 
18
20
  class ShellCompleter(Completer):
19
21
  """
@@ -30,7 +32,6 @@ class ShellCompleter(Completer):
30
32
  yielded = []
31
33
  ensure_readline()
32
34
  parts = document.text.split('-')
33
- ends_with_space = parts[0].endswith(' ')
34
35
  last_action_line = parts[0].split('+')[-1]
35
36
  part_0_subbed_spaces = last_action_line.replace(' ', '_')
36
37
  parsed_text = (part_0_subbed_spaces + '-'.join(parts[1:]))
@@ -130,12 +130,15 @@ def _ask_with_rowcounts(
130
130
  )
131
131
 
132
132
 
133
- pipes_rowcounts = {p: p.get_rowcount(begin=begin, end=end, debug=debug) for p in pipes}
133
+ pipes_rowcounts = {
134
+ pipe: pipe.get_rowcount(begin=begin, end=end, debug=debug)
135
+ for pipe in pipes
136
+ }
134
137
  print_options(
135
- [str(p) + f'\n{rc}\n' for p, rc in pipes_rowcounts.items()],
138
+ [str(pipe) + f'\n{rowcount:,}\n' for pipe, rowcount in pipes_rowcounts.items()],
136
139
  header='Number of Rows to be Deleted'
137
140
  )
138
- total_num_rows = sum([rc for p, rc in pipes_rowcounts.items()])
141
+ total_num_rows = sum([rowcount for rowcount in pipes_rowcounts.values()])
139
142
  question = (
140
143
  f"Are you sure you want to delete {total_num_rows:,} rows across {len(pipes)} pipe"
141
144
  + ('s' if len(pipes) != 1 else '')
@@ -7,7 +7,7 @@ Functions for copying elements.
7
7
  """
8
8
 
9
9
  from __future__ import annotations
10
- from meerschaum.utils.typing import Union, Any, Sequence, SuccessTuple, Optional, Tuple, List
10
+ from meerschaum.utils.typing import Any, SuccessTuple, Optional, List
11
11
 
12
12
  def copy(
13
13
  action: Optional[List[str]] = None,
@@ -69,44 +69,53 @@ def _copy_pipes(
69
69
  Copy pipes' attributes and make new pipes.
70
70
  """
71
71
  from meerschaum import get_pipes, Pipe
72
- from meerschaum.utils.prompt import prompt, yes_no
72
+ from meerschaum.utils.prompt import prompt, yes_no, get_connectors_completer
73
73
  from meerschaum.utils.warnings import warn
74
74
  from meerschaum.utils.formatting import print_tuple
75
75
  from meerschaum.utils.formatting._shell import clear_screen
76
76
  pipes = get_pipes(as_list=True, **kw)
77
77
  successes = 0
78
- for p in pipes:
79
- ck = prompt(f"Connector keys for copy of {p}:", default=p.connector_keys)
80
- mk = prompt(f"Metric key for copy of {p}:", default=p.metric_key)
81
- lk = prompt(f"Location key for copy of {p} ('None' to omit):", default=str(p.location_key))
78
+ for pipe in pipes:
79
+ ck = prompt(
80
+ f"Connector keys for copy of {pipe}:",
81
+ default=pipe.connector_keys,
82
+ completer=get_connectors_completer(),
83
+ )
84
+ mk = prompt(f"Metric key for copy of {pipe}:", default=pipe.metric_key)
85
+ lk = prompt(
86
+ f"Location key for copy of {pipe} ('None' to omit):",
87
+ default=str(pipe.location_key),
88
+ )
82
89
  if lk in ('', 'None', '[None]'):
83
90
  lk = None
84
- _new_pipe = Pipe(
85
- ck, mk, lk,
86
- parameters=p.parameters.copy(),
87
- )
91
+
88
92
  instance_keys = prompt(
89
- f"Meerschaum instance to store the new {_new_pipe}:",
90
- default=p.instance_keys
93
+ f"Meerschaum instance for copy of {pipe}:",
94
+ default=pipe.instance_keys
91
95
  )
92
- _new_pipe.instance_keys = instance_keys
93
- if _new_pipe.get_id(debug=debug) is not None:
94
- warn(f"New {_new_pipe} already exists. Skipping...", stack=False)
96
+ new_pipe = Pipe(
97
+ ck, mk, lk,
98
+ instance=instance_keys,
99
+ parameters=pipe.parameters.copy(),
100
+ )
101
+
102
+ if new_pipe.get_id(debug=debug) is not None:
103
+ warn(f"{new_pipe} already exists. Skipping...", stack=False)
95
104
  continue
96
- _register_success_tuple = _new_pipe.register(debug=debug)
105
+ _register_success_tuple = new_pipe.register(debug=debug)
97
106
  if not _register_success_tuple[0]:
98
- warn(f"Failed to register new {_new_pipe}.", stack=False)
107
+ warn(f"Failed to register new {new_pipe}.", stack=False)
99
108
  continue
100
109
 
101
110
  clear_screen(debug=debug)
102
111
  successes += 1
103
112
  print_tuple(
104
- (True, f"Successfully copied attributes of {p} " + f" into {_new_pipe}.")
113
+ (True, f"Successfully copied attributes of {pipe} " + f" into {new_pipe}.")
105
114
  )
106
115
  if (
107
116
  force or yes_no(
108
117
  (
109
- f"Do you want to copy data from {p} into {_new_pipe}?\n\n"
118
+ f"Do you want to copy data from {pipe} into {new_pipe}?\n\n"
110
119
  + "If you specified `--begin`, `--end` or `--params`, data will be filtered."
111
120
  ),
112
121
  noask=noask,
@@ -114,8 +123,8 @@ def _copy_pipes(
114
123
  default='n',
115
124
  )
116
125
  ):
117
- _new_pipe.sync(
118
- p.get_data(
126
+ new_pipe.sync(
127
+ pipe.get_data(
119
128
  debug=debug,
120
129
  as_iterator=True,
121
130
  **kw
@@ -142,17 +151,14 @@ def _copy_connectors(
142
151
  ) -> SuccessTuple:
143
152
  """
144
153
  Create a new connector from an existing one.
145
-
146
154
  """
147
- import os, pathlib
148
155
  from meerschaum.utils.prompt import yes_no, prompt
149
156
  from meerschaum.connectors.parse import parse_connector_keys
150
157
  from meerschaum.config import _config, get_config
151
- from meerschaum.config._edit import write_config
152
- from meerschaum.utils.warnings import info, warn
158
+ from meerschaum.utils.warnings import info
153
159
  from meerschaum.utils.formatting import pprint
154
160
  from meerschaum.actions import get_action
155
- cf = _config()
161
+ _ = _config()
156
162
  if action is None:
157
163
  action = []
158
164
  if connector_keys is None:
@@ -170,7 +176,7 @@ def _copy_connectors(
170
176
 
171
177
  try:
172
178
  conn = parse_connector_keys(ck)
173
- except Exception as e:
179
+ except Exception:
174
180
  return False, f"Unable to parse connector '{ck}'."
175
181
 
176
182
  if len(_keys) == 2:
meerschaum/actions/sql.py CHANGED
@@ -21,7 +21,7 @@ def sql(
21
21
  nopretty: bool = False,
22
22
  debug: bool = False,
23
23
  **kw: Any
24
- ):
24
+ ) -> SuccessTuple:
25
25
  """Execute a SQL query or launch an interactive CLI. All positional arguments are optional.
26
26
 
27
27
  Usage:
@@ -125,10 +125,9 @@ def sql(
125
125
  if result is False:
126
126
  return (False, f"Failed to execute query:\n\n{query}")
127
127
 
128
- from meerschaum.utils.packages import attempt_import, import_pandas
128
+ from meerschaum.utils.packages import attempt_import
129
129
  from meerschaum.utils.formatting import print_tuple, pprint
130
- sqlalchemy_engine_result = attempt_import('sqlalchemy.engine.result')
131
- pd = import_pandas()
130
+ _ = attempt_import('sqlalchemy.engine.result')
132
131
  if 'sqlalchemy' in str(type(result)):
133
132
  if not nopretty:
134
133
  print_tuple((True, f"Successfully executed query:\n\n{query}"))
@@ -145,3 +144,14 @@ def sql(
145
144
  )
146
145
 
147
146
  return (True, "Success")
147
+
148
+
149
+ def _complete_sql(
150
+ action: Optional[List[str]] = None, **kw: Any
151
+ ) -> List[str]:
152
+ from meerschaum.utils.misc import get_connector_labels
153
+ _text = action[0] if action else ""
154
+ return [
155
+ label.split('sql:', maxsplit=1)[-1]
156
+ for label in get_connector_labels('sql', search_term=_text, ignore_exact_match=True)
157
+ ]
@@ -275,12 +275,14 @@ def _sync_pipes(
275
275
  import time
276
276
  import os
277
277
  import contextlib
278
+ from datetime import timedelta
278
279
 
279
280
  from meerschaum.utils.warnings import warn, info
280
281
  from meerschaum.utils.formatting._shell import progress
281
282
  from meerschaum.utils.formatting._shell import clear_screen
282
283
  from meerschaum.utils.formatting import print_pipes_results
283
284
  from meerschaum.config.static import STATIC_CONFIG
285
+ from meerschaum.utils.misc import interval_str
284
286
 
285
287
  noninteractive_val = os.environ.get(STATIC_CONFIG['environment']['noninteractive'], None)
286
288
  noninteractive = str(noninteractive_val).lower() in ('1', 'true', 'yes')
@@ -321,7 +323,7 @@ def _sync_pipes(
321
323
  for pipe, (_success, _msg) in results_dict.items()
322
324
  if not _success
323
325
  ]
324
- except Exception as e:
326
+ except Exception:
325
327
  import traceback
326
328
  traceback.print_exc()
327
329
  warn(
@@ -351,9 +353,9 @@ def _sync_pipes(
351
353
  success_msg = (
352
354
  "Successfully spawned threads for pipes:"
353
355
  if unblock
354
- else f"Successfully synced pipes:"
356
+ else "Successfully synced pipes:"
355
357
  )
356
- fail_msg = f"Failed to sync pipes:"
358
+ fail_msg = "Failed to sync pipes:"
357
359
  if results_dict:
358
360
  print_pipes_results(
359
361
  results_dict,
@@ -362,8 +364,10 @@ def _sync_pipes(
362
364
  nopretty = nopretty,
363
365
  )
364
366
 
367
+ lap_duration_text = interval_str(timedelta(seconds=(lap_end - lap_begin)))
368
+
365
369
  msg = (
366
- f"It took {round(lap_end - lap_begin, 2)} seconds to sync " +
370
+ f"It took {lap_duration_text} to sync " +
367
371
  f"{len(success_pipes) + len(failure_pipes)} pipe" +
368
372
  ("s" if (len(success_pipes) + len(failure_pipes)) != 1 else "") + "\n" +
369
373
  f" ({len(success_pipes)} succeeded, {len(failure_pipes)} failed)."
@@ -385,26 +389,26 @@ def _sync_pipes(
385
389
 
386
390
 
387
391
  def _wrap_pipe(
388
- pipe,
389
- unblock: bool = False,
390
- force: bool = False,
391
- debug: bool = False,
392
- min_seconds: int = 1,
393
- workers = None,
394
- verify: bool = False,
395
- deduplicate: bool = False,
396
- bounded: Optional[bool] = None,
397
- chunk_interval: Union[timedelta, int, None] = None,
398
- **kw
399
- ):
392
+ pipe,
393
+ unblock: bool = False,
394
+ force: bool = False,
395
+ debug: bool = False,
396
+ min_seconds: int = 1,
397
+ workers = None,
398
+ verify: bool = False,
399
+ deduplicate: bool = False,
400
+ bounded: Optional[bool] = None,
401
+ chunk_interval: Union[timedelta, int, None] = None,
402
+ **kw
403
+ ):
400
404
  """
401
405
  Wrapper function for handling exceptions.
402
406
  """
403
407
  import time
404
408
  import traceback
405
- from datetime import datetime, timedelta, timezone
409
+ from datetime import datetime, timezone
406
410
  import meerschaum as mrsm
407
- from meerschaum.utils.typing import is_success_tuple, SuccessTuple
411
+ from meerschaum.utils.typing import is_success_tuple
408
412
  from meerschaum.connectors import get_connector_plugin
409
413
  from meerschaum.utils.venv import Venv
410
414
  from meerschaum.plugins import _pre_sync_hooks, _post_sync_hooks
@@ -12,7 +12,6 @@ import shlex
12
12
  from textwrap import dedent
13
13
  from urllib.parse import urlencode
14
14
 
15
- from dash.dependencies import Input, Output, State
16
15
  from meerschaum.utils.typing import List, Optional, Dict, Any, Tuple, Union
17
16
  from meerschaum.utils.misc import string_to_dict
18
17
  from meerschaum.utils.packages import attempt_import, import_dcc, import_html, import_pandas
@@ -76,7 +75,7 @@ def keys_from_state(
76
75
  try:
77
76
  # params = string_to_dict(state['params-textarea.value'])
78
77
  params = string_to_dict(state['search-parameters-editor.value'])
79
- except Exception as e:
78
+ except Exception:
80
79
  params = None
81
80
  else:
82
81
  params = None
@@ -506,7 +505,7 @@ def accordion_items_from_pipe(
506
505
 
507
506
  stats_rows = []
508
507
  if rowcount is not None:
509
- stats_rows.append(html.Tr([html.Td("Row-count"), html.Td(f"{rowcount}")]))
508
+ stats_rows.append(html.Tr([html.Td("Row Count"), html.Td(f"{rowcount:,}")]))
510
509
  if interval is not None:
511
510
  stats_rows.append(
512
511
  html.Tr([html.Td("Timespan"), html.Td(humanfriendly.format_timespan(interval))])
@@ -68,8 +68,19 @@ default_system_config = {
68
68
  'pandas': 'pandas',
69
69
  },
70
70
  'sql': {
71
+ 'bulk_insert': {
72
+ 'postgresql': True,
73
+ 'citus': True,
74
+ 'timescaledb': True,
75
+ 'mssql': True,
76
+ },
71
77
  'instance': {
72
78
  'stale_temporary_tables_minutes': 1440,
79
+ 'temporary_target': {
80
+ 'prefix': '_',
81
+ 'transaction_id_length': 4,
82
+ 'separator': '_',
83
+ },
73
84
  },
74
85
  'chunksize': 100_000,
75
86
  'poolclass': 'sqlalchemy.pool.QueuePool',
@@ -2,4 +2,4 @@
2
2
  Specify the Meerschaum release version.
3
3
  """
4
4
 
5
- __version__ = "2.7.4"
5
+ __version__ = "2.7.6"
@@ -20,12 +20,13 @@ def get_mrsm_version(self, **kw) -> Optional[str]:
20
20
  use_token=False,
21
21
  **kw
22
22
  ).json()
23
- except Exception as e:
23
+ except Exception:
24
24
  return None
25
25
  if isinstance(j, dict) and 'detail' in j:
26
26
  return None
27
27
  return j
28
28
 
29
+
29
30
  def get_chaining_status(self, **kw) -> Optional[bool]:
30
31
  """
31
32
  Fetch the chaining status of the API instance.
@@ -39,7 +40,7 @@ def get_chaining_status(self, **kw) -> Optional[bool]:
39
40
  )
40
41
  if not response:
41
42
  return None
42
- except Exception as e:
43
+ except Exception:
43
44
  return None
44
45
 
45
46
  return response.json()
@@ -9,8 +9,7 @@ Register or fetch Pipes from the API
9
9
  from __future__ import annotations
10
10
  import time
11
11
  import json
12
- from io import StringIO
13
- from datetime import datetime
12
+ from datetime import datetime, timedelta
14
13
 
15
14
  import meerschaum as mrsm
16
15
  from meerschaum.utils.debug import dprint
@@ -178,11 +177,11 @@ def sync_pipe(
178
177
  """Sync a DataFrame into a Pipe."""
179
178
  from decimal import Decimal
180
179
  from meerschaum.utils.debug import dprint
181
- from meerschaum.utils.misc import json_serialize_datetime, items_str
180
+ from meerschaum.utils.misc import json_serialize_datetime, items_str, interval_str
182
181
  from meerschaum.config import get_config
183
182
  from meerschaum.utils.packages import attempt_import
184
- from meerschaum.utils.dataframe import get_numeric_cols, to_json, get_bytes_cols
185
- begin = time.time()
183
+ from meerschaum.utils.dataframe import get_numeric_cols, to_json
184
+ begin = time.perf_counter()
186
185
  more_itertools = attempt_import('more_itertools')
187
186
  if df is None:
188
187
  msg = f"DataFrame is `None`. Cannot sync {pipe}."
@@ -304,9 +303,10 @@ def sync_pipe(
304
303
  num_success_chunks += 1
305
304
 
306
305
  success_tuple = True, (
307
- f"It took {round(time.time() - begin, 2)} seconds to sync {rowcount} row"
306
+ f"It took {interval_str(timedelta(seconds=(time.perf_counter() - begin)))} "
307
+ + "to sync {rowcount:,} row"
308
308
  + ('s' if rowcount != 1 else '')
309
- + f" across {num_success_chunks} chunk" + ('s' if num_success_chunks != 1 else '') +
309
+ + f" across {num_success_chunks:,} chunk" + ('s' if num_success_chunks != 1 else '') +
310
310
  f" to {pipe}."
311
311
  )
312
312
  return success_tuple
@@ -549,10 +549,9 @@ def create_metadata(
549
549
  if debug:
550
550
  dprint("Create metadata response: {response.text}")
551
551
  try:
552
- metadata_response = json.loads(response.text)
552
+ _ = json.loads(response.text)
553
553
  except Exception as e:
554
554
  warn(f"Failed to create metadata on {self}:\n{e}")
555
- metadata_response = False
556
555
  return False
557
556
 
558
557
 
@@ -69,6 +69,7 @@ class SQLConnector(Connector):
69
69
  get_pipe_schema,
70
70
  create_pipe_table_from_df,
71
71
  get_pipe_columns_indices,
72
+ get_temporary_target,
72
73
  )
73
74
  from ._plugins import (
74
75
  register_plugin,
@@ -26,46 +26,52 @@ flavor_clis = {
26
26
  'duckdb' : 'gadwall',
27
27
  }
28
28
  cli_deps = {
29
- 'pgcli': ['pgspecial', 'pendulum'],
29
+ 'pgcli': ['pgspecial', 'pendulum', 'cli_helpers'],
30
30
  'mycli': ['cryptography'],
31
+ 'mssql': ['cli_helpers'],
31
32
  }
32
33
 
33
34
 
34
35
  def cli(
35
- self,
36
- debug: bool = False,
37
- ) -> SuccessTuple:
36
+ self,
37
+ debug: bool = False,
38
+ ) -> SuccessTuple:
38
39
  """
39
40
  Launch a subprocess for an interactive CLI.
40
41
  """
42
+ from meerschaum.utils.warnings import dprint
41
43
  from meerschaum.utils.venv import venv_exec
42
44
  env = copy.deepcopy(dict(os.environ))
43
- env[f'MRSM_SQL_{self.label.upper()}'] = json.dumps(self.meta)
45
+ env_key = f"MRSM_SQL_{self.label.upper()}"
46
+ env_val = json.dumps(self.meta)
47
+ env[env_key] = env_val
44
48
  cli_code = (
45
49
  "import sys\n"
46
50
  "import meerschaum as mrsm\n"
51
+ "import os\n"
47
52
  f"conn = mrsm.get_connector('sql:{self.label}')\n"
48
53
  "success, msg = conn._cli_exit()\n"
49
54
  "mrsm.pprint((success, msg))\n"
50
55
  "if not success:\n"
51
56
  " raise Exception(msg)"
52
57
  )
58
+ if debug:
59
+ dprint(cli_code)
53
60
  try:
54
- _ = venv_exec(cli_code, venv=None, debug=debug, capture_output=False)
61
+ _ = venv_exec(cli_code, venv=None, env=env, debug=debug, capture_output=False)
55
62
  except Exception as e:
56
63
  return False, f"[{self}] Failed to start CLI:\n{e}"
57
64
  return True, "Success"
58
65
 
59
66
 
60
67
  def _cli_exit(
61
- self,
62
- debug: bool = False
63
- ) -> SuccessTuple:
68
+ self,
69
+ debug: bool = False
70
+ ) -> SuccessTuple:
64
71
  """Launch an interactive CLI for the SQLConnector's flavor."""
65
- from meerschaum.utils.packages import venv_exec, attempt_import
72
+ import os
73
+ from meerschaum.utils.packages import attempt_import
66
74
  from meerschaum.utils.debug import dprint
67
- from meerschaum.utils.warnings import error
68
- import sys, subprocess, os
69
75
 
70
76
  if self.flavor not in flavor_clis:
71
77
  return False, f"No CLI available for flavor '{self.flavor}'."
@@ -61,6 +61,7 @@ flavor_configs = {
61
61
  'engine': 'mssql+pyodbc',
62
62
  'create_engine': {
63
63
  'fast_executemany': True,
64
+ 'use_insertmanyvalues': False,
64
65
  'isolation_level': 'AUTOCOMMIT',
65
66
  'use_setinputsizes': False,
66
67
  'pool_pre_ping': True,
@@ -1505,7 +1505,6 @@ def sync_pipe(
1505
1505
  UPDATE_QUERIES,
1506
1506
  get_reset_autoincrement_queries,
1507
1507
  )
1508
- from meerschaum.utils.misc import generate_password
1509
1508
  from meerschaum.utils.dtypes import are_dtypes_equal
1510
1509
  from meerschaum.utils.dtypes.sql import get_db_type_from_pd_type
1511
1510
  from meerschaum import Pipe
@@ -1720,9 +1719,7 @@ def sync_pipe(
1720
1719
  warn(f"Could not reset auto-incrementing primary key for {pipe}.", stack=False)
1721
1720
 
1722
1721
  if update_df is not None and len(update_df) > 0:
1723
- transact_id = generate_password(3)
1724
- temp_prefix = '##' if self.flavor != 'oracle' else '_'
1725
- temp_target = temp_prefix + transact_id + '_' + pipe.target
1722
+ temp_target = self.get_temporary_target(pipe.target, label='update')
1726
1723
  self._log_temporary_tables_creation(temp_target, create=(not pipe.temporary), debug=debug)
1727
1724
  temp_pipe = Pipe(
1728
1725
  pipe.connector_keys.replace(':', '_') + '_', pipe.metric_key, pipe.location_key,
@@ -1743,7 +1740,7 @@ def sync_pipe(
1743
1740
  static=True,
1744
1741
  autoincrement=False,
1745
1742
  parameters={
1746
- 'schema': (self.internal_schema if self.flavor != 'mssql' else None),
1743
+ 'schema': self.internal_schema,
1747
1744
  'hypertable': False,
1748
1745
  },
1749
1746
  )
@@ -1910,15 +1907,18 @@ def sync_pipe_inplace(
1910
1907
  )
1911
1908
  from meerschaum.utils.misc import generate_password
1912
1909
 
1913
- transact_id = generate_password(3)
1914
- def get_temp_table_name(label: str) -> str:
1915
- temp_prefix = '##' if self.flavor != 'oracle' else '_'
1916
- return temp_prefix + transact_id + '_' + label + '_' + pipe.target
1910
+ transaction_id_length = (
1911
+ mrsm.get_config(
1912
+ 'system', 'connectors', 'sql', 'instance', 'temporary_target', 'transaction_id_length'
1913
+ )
1914
+ )
1915
+ transact_id = generate_password(transaction_id_length)
1917
1916
 
1918
1917
  internal_schema = self.internal_schema
1918
+ target = pipe.target
1919
1919
  temp_table_roots = ['backtrack', 'new', 'delta', 'joined', 'unseen', 'update']
1920
1920
  temp_tables = {
1921
- table_root: get_temp_table_name(table_root)
1921
+ table_root: self.get_temporary_target(target, transact_id=transact_id, label=table_root)
1922
1922
  for table_root in temp_table_roots
1923
1923
  }
1924
1924
  temp_table_names = {
@@ -3064,7 +3064,12 @@ def get_alter_columns_queries(
3064
3064
  return []
3065
3065
  if pipe.static:
3066
3066
  return
3067
- from meerschaum.utils.sql import sql_item_name, DROP_IF_EXISTS_FLAVORS, get_table_cols_types
3067
+ from meerschaum.utils.sql import (
3068
+ sql_item_name,
3069
+ get_table_cols_types,
3070
+ DROP_IF_EXISTS_FLAVORS,
3071
+ SINGLE_ALTER_TABLE_FLAVORS,
3072
+ )
3068
3073
  from meerschaum.utils.dataframe import get_numeric_cols
3069
3074
  from meerschaum.utils.dtypes import are_dtypes_equal
3070
3075
  from meerschaum.utils.dtypes.sql import (
@@ -3308,14 +3313,19 @@ def get_alter_columns_queries(
3308
3313
  else 'TYPE '
3309
3314
  )
3310
3315
  column_str = 'COLUMN' if self.flavor != 'oracle' else ''
3311
- query += (
3316
+ query_suffix = (
3312
3317
  f"\n{alter_col_prefix} {column_str} "
3313
3318
  + sql_item_name(col, self.flavor, None)
3314
3319
  + " " + type_prefix + typ + ","
3315
3320
  )
3321
+ if self.flavor not in SINGLE_ALTER_TABLE_FLAVORS:
3322
+ query += query_suffix
3323
+ else:
3324
+ queries.append(query + query_suffix[:-1])
3325
+
3326
+ if self.flavor not in SINGLE_ALTER_TABLE_FLAVORS:
3327
+ queries.append(query[:-1])
3316
3328
 
3317
- query = query[:-1]
3318
- queries.append(query)
3319
3329
  if self.flavor != 'duckdb':
3320
3330
  return queries
3321
3331
 
@@ -3616,3 +3626,30 @@ def get_pipe_schema(self, pipe: mrsm.Pipe) -> Union[str, None]:
3616
3626
  A schema string or `None` if nothing is configured.
3617
3627
  """
3618
3628
  return pipe.parameters.get('schema', self.schema)
3629
+
3630
+
3631
+ @staticmethod
3632
+ def get_temporary_target(
3633
+ target: str,
3634
+ transact_id: Optional[str, None] = None,
3635
+ label: Optional[str] = None,
3636
+ separator: Optional[str] = None,
3637
+ ) -> str:
3638
+ """
3639
+ Return a unique(ish) temporary target for a pipe.
3640
+ """
3641
+ from meerschaum.utils.misc import generate_password
3642
+ temp_target_cf = (
3643
+ mrsm.get_config('system', 'connectors', 'sql', 'instance', 'temporary_target') or {}
3644
+ )
3645
+ transaction_id_len = temp_target_cf.get('transaction_id_length', 3)
3646
+ transact_id = transact_id or generate_password(transaction_id_len)
3647
+ temp_prefix = temp_target_cf.get('prefix', '_')
3648
+ separator = separator or temp_target_cf.get('separator', '_')
3649
+ return (
3650
+ temp_prefix
3651
+ + target
3652
+ + separator
3653
+ + transact_id
3654
+ + ((separator + label) if label else '')
3655
+ )