meerschaum 2.7.4__py3-none-any.whl → 2.7.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/_internal/shell/Shell.py +4 -6
- meerschaum/_internal/shell/ShellCompleter.py +6 -5
- meerschaum/actions/clear.py +6 -3
- meerschaum/actions/copy.py +33 -27
- meerschaum/actions/sql.py +14 -4
- meerschaum/actions/sync.py +22 -18
- meerschaum/api/dash/pipes.py +2 -3
- meerschaum/config/_default.py +11 -0
- meerschaum/config/_version.py +1 -1
- meerschaum/connectors/api/_misc.py +3 -2
- meerschaum/connectors/api/_pipes.py +8 -9
- meerschaum/connectors/sql/_SQLConnector.py +1 -0
- meerschaum/connectors/sql/_cli.py +18 -12
- meerschaum/connectors/sql/_create_engine.py +1 -0
- meerschaum/connectors/sql/_pipes.py +51 -14
- meerschaum/connectors/sql/_sql.py +109 -16
- meerschaum/jobs/_Job.py +1 -0
- meerschaum/plugins/__init__.py +7 -3
- meerschaum/utils/daemon/Daemon.py +11 -3
- meerschaum/utils/daemon/__init__.py +2 -2
- meerschaum/utils/misc.py +7 -6
- meerschaum/utils/packages/__init__.py +35 -28
- meerschaum/utils/packages/_packages.py +1 -1
- meerschaum/utils/prompt.py +54 -36
- meerschaum/utils/venv/_Venv.py +6 -1
- meerschaum/utils/venv/__init__.py +32 -16
- {meerschaum-2.7.4.dist-info → meerschaum-2.7.6.dist-info}/METADATA +4 -4
- {meerschaum-2.7.4.dist-info → meerschaum-2.7.6.dist-info}/RECORD +34 -34
- {meerschaum-2.7.4.dist-info → meerschaum-2.7.6.dist-info}/WHEEL +1 -1
- {meerschaum-2.7.4.dist-info → meerschaum-2.7.6.dist-info}/LICENSE +0 -0
- {meerschaum-2.7.4.dist-info → meerschaum-2.7.6.dist-info}/NOTICE +0 -0
- {meerschaum-2.7.4.dist-info → meerschaum-2.7.6.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.7.4.dist-info → meerschaum-2.7.6.dist-info}/top_level.txt +0 -0
- {meerschaum-2.7.4.dist-info → meerschaum-2.7.6.dist-info}/zip-safe +0 -0
@@ -742,8 +742,7 @@ class Shell(cmd.Cmd):
|
|
742
742
|
"""
|
743
743
|
from meerschaum import get_connector
|
744
744
|
from meerschaum.connectors.parse import parse_instance_keys
|
745
|
-
from meerschaum.utils.warnings import
|
746
|
-
from meerschaum.utils.misc import remove_ansi
|
745
|
+
from meerschaum.utils.warnings import info
|
747
746
|
|
748
747
|
if action is None:
|
749
748
|
action = []
|
@@ -829,7 +828,7 @@ class Shell(cmd.Cmd):
|
|
829
828
|
"""
|
830
829
|
from meerschaum import get_connector
|
831
830
|
from meerschaum.connectors.parse import parse_repo_keys
|
832
|
-
from meerschaum.utils.warnings import
|
831
|
+
from meerschaum.utils.warnings import info
|
833
832
|
|
834
833
|
if action is None:
|
835
834
|
action = []
|
@@ -878,7 +877,6 @@ class Shell(cmd.Cmd):
|
|
878
877
|
|
879
878
|
Note that executors are API instances.
|
880
879
|
"""
|
881
|
-
from meerschaum import get_connector
|
882
880
|
from meerschaum.connectors.parse import parse_executor_keys
|
883
881
|
from meerschaum.utils.warnings import warn, info
|
884
882
|
from meerschaum.jobs import get_executor_keys_from_context
|
@@ -894,7 +892,7 @@ class Shell(cmd.Cmd):
|
|
894
892
|
executor_keys = get_executor_keys_from_context()
|
895
893
|
|
896
894
|
if executor_keys == 'systemd' and get_executor_keys_from_context() != 'systemd':
|
897
|
-
warn(
|
895
|
+
warn("Cannot execute via `systemd`, falling back to `local`...", stack=False)
|
898
896
|
executor_keys = 'local'
|
899
897
|
|
900
898
|
conn = parse_executor_keys(executor_keys, debug=debug)
|
@@ -935,7 +933,7 @@ class Shell(cmd.Cmd):
|
|
935
933
|
if args['action'][0] not in shell_attrs['_actions']:
|
936
934
|
try:
|
937
935
|
print(textwrap.dedent(getattr(self, f"do_{args['action'][0]}").__doc__))
|
938
|
-
except Exception
|
936
|
+
except Exception:
|
939
937
|
print(f"No help on '{args['action'][0]}'.")
|
940
938
|
return ""
|
941
939
|
parse_help(args)
|
@@ -7,13 +7,15 @@ Implement the prompt_toolkit Completer base class.
|
|
7
7
|
"""
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
|
-
from
|
11
|
-
from meerschaum.utils.typing import Optional
|
12
|
-
from meerschaum.actions import get_shell, get_completer, get_main_action_name, get_action
|
10
|
+
from meerschaum.actions import get_shell, get_main_action_name, get_action
|
13
11
|
from meerschaum._internal.arguments import parse_line
|
14
12
|
|
15
13
|
from meerschaum.utils.packages import attempt_import, ensure_readline
|
16
|
-
|
14
|
+
|
15
|
+
prompt_toolkit_completion = attempt_import('prompt_toolkit.completion', lazy=False, install=True)
|
16
|
+
Completer = prompt_toolkit_completion.Completer
|
17
|
+
Completion = prompt_toolkit_completion.Completion
|
18
|
+
|
17
19
|
|
18
20
|
class ShellCompleter(Completer):
|
19
21
|
"""
|
@@ -30,7 +32,6 @@ class ShellCompleter(Completer):
|
|
30
32
|
yielded = []
|
31
33
|
ensure_readline()
|
32
34
|
parts = document.text.split('-')
|
33
|
-
ends_with_space = parts[0].endswith(' ')
|
34
35
|
last_action_line = parts[0].split('+')[-1]
|
35
36
|
part_0_subbed_spaces = last_action_line.replace(' ', '_')
|
36
37
|
parsed_text = (part_0_subbed_spaces + '-'.join(parts[1:]))
|
meerschaum/actions/clear.py
CHANGED
@@ -130,12 +130,15 @@ def _ask_with_rowcounts(
|
|
130
130
|
)
|
131
131
|
|
132
132
|
|
133
|
-
pipes_rowcounts = {
|
133
|
+
pipes_rowcounts = {
|
134
|
+
pipe: pipe.get_rowcount(begin=begin, end=end, debug=debug)
|
135
|
+
for pipe in pipes
|
136
|
+
}
|
134
137
|
print_options(
|
135
|
-
[str(
|
138
|
+
[str(pipe) + f'\n{rowcount:,}\n' for pipe, rowcount in pipes_rowcounts.items()],
|
136
139
|
header='Number of Rows to be Deleted'
|
137
140
|
)
|
138
|
-
total_num_rows = sum([
|
141
|
+
total_num_rows = sum([rowcount for rowcount in pipes_rowcounts.values()])
|
139
142
|
question = (
|
140
143
|
f"Are you sure you want to delete {total_num_rows:,} rows across {len(pipes)} pipe"
|
141
144
|
+ ('s' if len(pipes) != 1 else '')
|
meerschaum/actions/copy.py
CHANGED
@@ -7,7 +7,7 @@ Functions for copying elements.
|
|
7
7
|
"""
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
|
-
from meerschaum.utils.typing import
|
10
|
+
from meerschaum.utils.typing import Any, SuccessTuple, Optional, List
|
11
11
|
|
12
12
|
def copy(
|
13
13
|
action: Optional[List[str]] = None,
|
@@ -69,44 +69,53 @@ def _copy_pipes(
|
|
69
69
|
Copy pipes' attributes and make new pipes.
|
70
70
|
"""
|
71
71
|
from meerschaum import get_pipes, Pipe
|
72
|
-
from meerschaum.utils.prompt import prompt, yes_no
|
72
|
+
from meerschaum.utils.prompt import prompt, yes_no, get_connectors_completer
|
73
73
|
from meerschaum.utils.warnings import warn
|
74
74
|
from meerschaum.utils.formatting import print_tuple
|
75
75
|
from meerschaum.utils.formatting._shell import clear_screen
|
76
76
|
pipes = get_pipes(as_list=True, **kw)
|
77
77
|
successes = 0
|
78
|
-
for
|
79
|
-
ck = prompt(
|
80
|
-
|
81
|
-
|
78
|
+
for pipe in pipes:
|
79
|
+
ck = prompt(
|
80
|
+
f"Connector keys for copy of {pipe}:",
|
81
|
+
default=pipe.connector_keys,
|
82
|
+
completer=get_connectors_completer(),
|
83
|
+
)
|
84
|
+
mk = prompt(f"Metric key for copy of {pipe}:", default=pipe.metric_key)
|
85
|
+
lk = prompt(
|
86
|
+
f"Location key for copy of {pipe} ('None' to omit):",
|
87
|
+
default=str(pipe.location_key),
|
88
|
+
)
|
82
89
|
if lk in ('', 'None', '[None]'):
|
83
90
|
lk = None
|
84
|
-
|
85
|
-
ck, mk, lk,
|
86
|
-
parameters=p.parameters.copy(),
|
87
|
-
)
|
91
|
+
|
88
92
|
instance_keys = prompt(
|
89
|
-
f"Meerschaum instance
|
90
|
-
default=
|
93
|
+
f"Meerschaum instance for copy of {pipe}:",
|
94
|
+
default=pipe.instance_keys
|
91
95
|
)
|
92
|
-
|
93
|
-
|
94
|
-
|
96
|
+
new_pipe = Pipe(
|
97
|
+
ck, mk, lk,
|
98
|
+
instance=instance_keys,
|
99
|
+
parameters=pipe.parameters.copy(),
|
100
|
+
)
|
101
|
+
|
102
|
+
if new_pipe.get_id(debug=debug) is not None:
|
103
|
+
warn(f"{new_pipe} already exists. Skipping...", stack=False)
|
95
104
|
continue
|
96
|
-
_register_success_tuple =
|
105
|
+
_register_success_tuple = new_pipe.register(debug=debug)
|
97
106
|
if not _register_success_tuple[0]:
|
98
|
-
warn(f"Failed to register new {
|
107
|
+
warn(f"Failed to register new {new_pipe}.", stack=False)
|
99
108
|
continue
|
100
109
|
|
101
110
|
clear_screen(debug=debug)
|
102
111
|
successes += 1
|
103
112
|
print_tuple(
|
104
|
-
(True, f"Successfully copied attributes of {
|
113
|
+
(True, f"Successfully copied attributes of {pipe} " + f" into {new_pipe}.")
|
105
114
|
)
|
106
115
|
if (
|
107
116
|
force or yes_no(
|
108
117
|
(
|
109
|
-
f"Do you want to copy data from {
|
118
|
+
f"Do you want to copy data from {pipe} into {new_pipe}?\n\n"
|
110
119
|
+ "If you specified `--begin`, `--end` or `--params`, data will be filtered."
|
111
120
|
),
|
112
121
|
noask=noask,
|
@@ -114,8 +123,8 @@ def _copy_pipes(
|
|
114
123
|
default='n',
|
115
124
|
)
|
116
125
|
):
|
117
|
-
|
118
|
-
|
126
|
+
new_pipe.sync(
|
127
|
+
pipe.get_data(
|
119
128
|
debug=debug,
|
120
129
|
as_iterator=True,
|
121
130
|
**kw
|
@@ -142,17 +151,14 @@ def _copy_connectors(
|
|
142
151
|
) -> SuccessTuple:
|
143
152
|
"""
|
144
153
|
Create a new connector from an existing one.
|
145
|
-
|
146
154
|
"""
|
147
|
-
import os, pathlib
|
148
155
|
from meerschaum.utils.prompt import yes_no, prompt
|
149
156
|
from meerschaum.connectors.parse import parse_connector_keys
|
150
157
|
from meerschaum.config import _config, get_config
|
151
|
-
from meerschaum.
|
152
|
-
from meerschaum.utils.warnings import info, warn
|
158
|
+
from meerschaum.utils.warnings import info
|
153
159
|
from meerschaum.utils.formatting import pprint
|
154
160
|
from meerschaum.actions import get_action
|
155
|
-
|
161
|
+
_ = _config()
|
156
162
|
if action is None:
|
157
163
|
action = []
|
158
164
|
if connector_keys is None:
|
@@ -170,7 +176,7 @@ def _copy_connectors(
|
|
170
176
|
|
171
177
|
try:
|
172
178
|
conn = parse_connector_keys(ck)
|
173
|
-
except Exception
|
179
|
+
except Exception:
|
174
180
|
return False, f"Unable to parse connector '{ck}'."
|
175
181
|
|
176
182
|
if len(_keys) == 2:
|
meerschaum/actions/sql.py
CHANGED
@@ -21,7 +21,7 @@ def sql(
|
|
21
21
|
nopretty: bool = False,
|
22
22
|
debug: bool = False,
|
23
23
|
**kw: Any
|
24
|
-
):
|
24
|
+
) -> SuccessTuple:
|
25
25
|
"""Execute a SQL query or launch an interactive CLI. All positional arguments are optional.
|
26
26
|
|
27
27
|
Usage:
|
@@ -125,10 +125,9 @@ def sql(
|
|
125
125
|
if result is False:
|
126
126
|
return (False, f"Failed to execute query:\n\n{query}")
|
127
127
|
|
128
|
-
from meerschaum.utils.packages import attempt_import
|
128
|
+
from meerschaum.utils.packages import attempt_import
|
129
129
|
from meerschaum.utils.formatting import print_tuple, pprint
|
130
|
-
|
131
|
-
pd = import_pandas()
|
130
|
+
_ = attempt_import('sqlalchemy.engine.result')
|
132
131
|
if 'sqlalchemy' in str(type(result)):
|
133
132
|
if not nopretty:
|
134
133
|
print_tuple((True, f"Successfully executed query:\n\n{query}"))
|
@@ -145,3 +144,14 @@ def sql(
|
|
145
144
|
)
|
146
145
|
|
147
146
|
return (True, "Success")
|
147
|
+
|
148
|
+
|
149
|
+
def _complete_sql(
|
150
|
+
action: Optional[List[str]] = None, **kw: Any
|
151
|
+
) -> List[str]:
|
152
|
+
from meerschaum.utils.misc import get_connector_labels
|
153
|
+
_text = action[0] if action else ""
|
154
|
+
return [
|
155
|
+
label.split('sql:', maxsplit=1)[-1]
|
156
|
+
for label in get_connector_labels('sql', search_term=_text, ignore_exact_match=True)
|
157
|
+
]
|
meerschaum/actions/sync.py
CHANGED
@@ -275,12 +275,14 @@ def _sync_pipes(
|
|
275
275
|
import time
|
276
276
|
import os
|
277
277
|
import contextlib
|
278
|
+
from datetime import timedelta
|
278
279
|
|
279
280
|
from meerschaum.utils.warnings import warn, info
|
280
281
|
from meerschaum.utils.formatting._shell import progress
|
281
282
|
from meerschaum.utils.formatting._shell import clear_screen
|
282
283
|
from meerschaum.utils.formatting import print_pipes_results
|
283
284
|
from meerschaum.config.static import STATIC_CONFIG
|
285
|
+
from meerschaum.utils.misc import interval_str
|
284
286
|
|
285
287
|
noninteractive_val = os.environ.get(STATIC_CONFIG['environment']['noninteractive'], None)
|
286
288
|
noninteractive = str(noninteractive_val).lower() in ('1', 'true', 'yes')
|
@@ -321,7 +323,7 @@ def _sync_pipes(
|
|
321
323
|
for pipe, (_success, _msg) in results_dict.items()
|
322
324
|
if not _success
|
323
325
|
]
|
324
|
-
except Exception
|
326
|
+
except Exception:
|
325
327
|
import traceback
|
326
328
|
traceback.print_exc()
|
327
329
|
warn(
|
@@ -351,9 +353,9 @@ def _sync_pipes(
|
|
351
353
|
success_msg = (
|
352
354
|
"Successfully spawned threads for pipes:"
|
353
355
|
if unblock
|
354
|
-
else
|
356
|
+
else "Successfully synced pipes:"
|
355
357
|
)
|
356
|
-
fail_msg =
|
358
|
+
fail_msg = "Failed to sync pipes:"
|
357
359
|
if results_dict:
|
358
360
|
print_pipes_results(
|
359
361
|
results_dict,
|
@@ -362,8 +364,10 @@ def _sync_pipes(
|
|
362
364
|
nopretty = nopretty,
|
363
365
|
)
|
364
366
|
|
367
|
+
lap_duration_text = interval_str(timedelta(seconds=(lap_end - lap_begin)))
|
368
|
+
|
365
369
|
msg = (
|
366
|
-
f"It took {
|
370
|
+
f"It took {lap_duration_text} to sync " +
|
367
371
|
f"{len(success_pipes) + len(failure_pipes)} pipe" +
|
368
372
|
("s" if (len(success_pipes) + len(failure_pipes)) != 1 else "") + "\n" +
|
369
373
|
f" ({len(success_pipes)} succeeded, {len(failure_pipes)} failed)."
|
@@ -385,26 +389,26 @@ def _sync_pipes(
|
|
385
389
|
|
386
390
|
|
387
391
|
def _wrap_pipe(
|
388
|
-
|
389
|
-
|
390
|
-
|
391
|
-
|
392
|
-
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
|
398
|
-
|
399
|
-
|
392
|
+
pipe,
|
393
|
+
unblock: bool = False,
|
394
|
+
force: bool = False,
|
395
|
+
debug: bool = False,
|
396
|
+
min_seconds: int = 1,
|
397
|
+
workers = None,
|
398
|
+
verify: bool = False,
|
399
|
+
deduplicate: bool = False,
|
400
|
+
bounded: Optional[bool] = None,
|
401
|
+
chunk_interval: Union[timedelta, int, None] = None,
|
402
|
+
**kw
|
403
|
+
):
|
400
404
|
"""
|
401
405
|
Wrapper function for handling exceptions.
|
402
406
|
"""
|
403
407
|
import time
|
404
408
|
import traceback
|
405
|
-
from datetime import datetime,
|
409
|
+
from datetime import datetime, timezone
|
406
410
|
import meerschaum as mrsm
|
407
|
-
from meerschaum.utils.typing import is_success_tuple
|
411
|
+
from meerschaum.utils.typing import is_success_tuple
|
408
412
|
from meerschaum.connectors import get_connector_plugin
|
409
413
|
from meerschaum.utils.venv import Venv
|
410
414
|
from meerschaum.plugins import _pre_sync_hooks, _post_sync_hooks
|
meerschaum/api/dash/pipes.py
CHANGED
@@ -12,7 +12,6 @@ import shlex
|
|
12
12
|
from textwrap import dedent
|
13
13
|
from urllib.parse import urlencode
|
14
14
|
|
15
|
-
from dash.dependencies import Input, Output, State
|
16
15
|
from meerschaum.utils.typing import List, Optional, Dict, Any, Tuple, Union
|
17
16
|
from meerschaum.utils.misc import string_to_dict
|
18
17
|
from meerschaum.utils.packages import attempt_import, import_dcc, import_html, import_pandas
|
@@ -76,7 +75,7 @@ def keys_from_state(
|
|
76
75
|
try:
|
77
76
|
# params = string_to_dict(state['params-textarea.value'])
|
78
77
|
params = string_to_dict(state['search-parameters-editor.value'])
|
79
|
-
except Exception
|
78
|
+
except Exception:
|
80
79
|
params = None
|
81
80
|
else:
|
82
81
|
params = None
|
@@ -506,7 +505,7 @@ def accordion_items_from_pipe(
|
|
506
505
|
|
507
506
|
stats_rows = []
|
508
507
|
if rowcount is not None:
|
509
|
-
stats_rows.append(html.Tr([html.Td("Row
|
508
|
+
stats_rows.append(html.Tr([html.Td("Row Count"), html.Td(f"{rowcount:,}")]))
|
510
509
|
if interval is not None:
|
511
510
|
stats_rows.append(
|
512
511
|
html.Tr([html.Td("Timespan"), html.Td(humanfriendly.format_timespan(interval))])
|
meerschaum/config/_default.py
CHANGED
@@ -68,8 +68,19 @@ default_system_config = {
|
|
68
68
|
'pandas': 'pandas',
|
69
69
|
},
|
70
70
|
'sql': {
|
71
|
+
'bulk_insert': {
|
72
|
+
'postgresql': True,
|
73
|
+
'citus': True,
|
74
|
+
'timescaledb': True,
|
75
|
+
'mssql': True,
|
76
|
+
},
|
71
77
|
'instance': {
|
72
78
|
'stale_temporary_tables_minutes': 1440,
|
79
|
+
'temporary_target': {
|
80
|
+
'prefix': '_',
|
81
|
+
'transaction_id_length': 4,
|
82
|
+
'separator': '_',
|
83
|
+
},
|
73
84
|
},
|
74
85
|
'chunksize': 100_000,
|
75
86
|
'poolclass': 'sqlalchemy.pool.QueuePool',
|
meerschaum/config/_version.py
CHANGED
@@ -20,12 +20,13 @@ def get_mrsm_version(self, **kw) -> Optional[str]:
|
|
20
20
|
use_token=False,
|
21
21
|
**kw
|
22
22
|
).json()
|
23
|
-
except Exception
|
23
|
+
except Exception:
|
24
24
|
return None
|
25
25
|
if isinstance(j, dict) and 'detail' in j:
|
26
26
|
return None
|
27
27
|
return j
|
28
28
|
|
29
|
+
|
29
30
|
def get_chaining_status(self, **kw) -> Optional[bool]:
|
30
31
|
"""
|
31
32
|
Fetch the chaining status of the API instance.
|
@@ -39,7 +40,7 @@ def get_chaining_status(self, **kw) -> Optional[bool]:
|
|
39
40
|
)
|
40
41
|
if not response:
|
41
42
|
return None
|
42
|
-
except Exception
|
43
|
+
except Exception:
|
43
44
|
return None
|
44
45
|
|
45
46
|
return response.json()
|
@@ -9,8 +9,7 @@ Register or fetch Pipes from the API
|
|
9
9
|
from __future__ import annotations
|
10
10
|
import time
|
11
11
|
import json
|
12
|
-
from
|
13
|
-
from datetime import datetime
|
12
|
+
from datetime import datetime, timedelta
|
14
13
|
|
15
14
|
import meerschaum as mrsm
|
16
15
|
from meerschaum.utils.debug import dprint
|
@@ -178,11 +177,11 @@ def sync_pipe(
|
|
178
177
|
"""Sync a DataFrame into a Pipe."""
|
179
178
|
from decimal import Decimal
|
180
179
|
from meerschaum.utils.debug import dprint
|
181
|
-
from meerschaum.utils.misc import json_serialize_datetime, items_str
|
180
|
+
from meerschaum.utils.misc import json_serialize_datetime, items_str, interval_str
|
182
181
|
from meerschaum.config import get_config
|
183
182
|
from meerschaum.utils.packages import attempt_import
|
184
|
-
from meerschaum.utils.dataframe import get_numeric_cols, to_json
|
185
|
-
begin = time.
|
183
|
+
from meerschaum.utils.dataframe import get_numeric_cols, to_json
|
184
|
+
begin = time.perf_counter()
|
186
185
|
more_itertools = attempt_import('more_itertools')
|
187
186
|
if df is None:
|
188
187
|
msg = f"DataFrame is `None`. Cannot sync {pipe}."
|
@@ -304,9 +303,10 @@ def sync_pipe(
|
|
304
303
|
num_success_chunks += 1
|
305
304
|
|
306
305
|
success_tuple = True, (
|
307
|
-
f"It took {
|
306
|
+
f"It took {interval_str(timedelta(seconds=(time.perf_counter() - begin)))} "
|
307
|
+
+ "to sync {rowcount:,} row"
|
308
308
|
+ ('s' if rowcount != 1 else '')
|
309
|
-
+ f" across {num_success_chunks} chunk" + ('s' if num_success_chunks != 1 else '') +
|
309
|
+
+ f" across {num_success_chunks:,} chunk" + ('s' if num_success_chunks != 1 else '') +
|
310
310
|
f" to {pipe}."
|
311
311
|
)
|
312
312
|
return success_tuple
|
@@ -549,10 +549,9 @@ def create_metadata(
|
|
549
549
|
if debug:
|
550
550
|
dprint("Create metadata response: {response.text}")
|
551
551
|
try:
|
552
|
-
|
552
|
+
_ = json.loads(response.text)
|
553
553
|
except Exception as e:
|
554
554
|
warn(f"Failed to create metadata on {self}:\n{e}")
|
555
|
-
metadata_response = False
|
556
555
|
return False
|
557
556
|
|
558
557
|
|
@@ -26,46 +26,52 @@ flavor_clis = {
|
|
26
26
|
'duckdb' : 'gadwall',
|
27
27
|
}
|
28
28
|
cli_deps = {
|
29
|
-
'pgcli': ['pgspecial', 'pendulum'],
|
29
|
+
'pgcli': ['pgspecial', 'pendulum', 'cli_helpers'],
|
30
30
|
'mycli': ['cryptography'],
|
31
|
+
'mssql': ['cli_helpers'],
|
31
32
|
}
|
32
33
|
|
33
34
|
|
34
35
|
def cli(
|
35
|
-
|
36
|
-
|
37
|
-
|
36
|
+
self,
|
37
|
+
debug: bool = False,
|
38
|
+
) -> SuccessTuple:
|
38
39
|
"""
|
39
40
|
Launch a subprocess for an interactive CLI.
|
40
41
|
"""
|
42
|
+
from meerschaum.utils.warnings import dprint
|
41
43
|
from meerschaum.utils.venv import venv_exec
|
42
44
|
env = copy.deepcopy(dict(os.environ))
|
43
|
-
|
45
|
+
env_key = f"MRSM_SQL_{self.label.upper()}"
|
46
|
+
env_val = json.dumps(self.meta)
|
47
|
+
env[env_key] = env_val
|
44
48
|
cli_code = (
|
45
49
|
"import sys\n"
|
46
50
|
"import meerschaum as mrsm\n"
|
51
|
+
"import os\n"
|
47
52
|
f"conn = mrsm.get_connector('sql:{self.label}')\n"
|
48
53
|
"success, msg = conn._cli_exit()\n"
|
49
54
|
"mrsm.pprint((success, msg))\n"
|
50
55
|
"if not success:\n"
|
51
56
|
" raise Exception(msg)"
|
52
57
|
)
|
58
|
+
if debug:
|
59
|
+
dprint(cli_code)
|
53
60
|
try:
|
54
|
-
_ = venv_exec(cli_code, venv=None, debug=debug, capture_output=False)
|
61
|
+
_ = venv_exec(cli_code, venv=None, env=env, debug=debug, capture_output=False)
|
55
62
|
except Exception as e:
|
56
63
|
return False, f"[{self}] Failed to start CLI:\n{e}"
|
57
64
|
return True, "Success"
|
58
65
|
|
59
66
|
|
60
67
|
def _cli_exit(
|
61
|
-
|
62
|
-
|
63
|
-
|
68
|
+
self,
|
69
|
+
debug: bool = False
|
70
|
+
) -> SuccessTuple:
|
64
71
|
"""Launch an interactive CLI for the SQLConnector's flavor."""
|
65
|
-
|
72
|
+
import os
|
73
|
+
from meerschaum.utils.packages import attempt_import
|
66
74
|
from meerschaum.utils.debug import dprint
|
67
|
-
from meerschaum.utils.warnings import error
|
68
|
-
import sys, subprocess, os
|
69
75
|
|
70
76
|
if self.flavor not in flavor_clis:
|
71
77
|
return False, f"No CLI available for flavor '{self.flavor}'."
|
@@ -1505,7 +1505,6 @@ def sync_pipe(
|
|
1505
1505
|
UPDATE_QUERIES,
|
1506
1506
|
get_reset_autoincrement_queries,
|
1507
1507
|
)
|
1508
|
-
from meerschaum.utils.misc import generate_password
|
1509
1508
|
from meerschaum.utils.dtypes import are_dtypes_equal
|
1510
1509
|
from meerschaum.utils.dtypes.sql import get_db_type_from_pd_type
|
1511
1510
|
from meerschaum import Pipe
|
@@ -1720,9 +1719,7 @@ def sync_pipe(
|
|
1720
1719
|
warn(f"Could not reset auto-incrementing primary key for {pipe}.", stack=False)
|
1721
1720
|
|
1722
1721
|
if update_df is not None and len(update_df) > 0:
|
1723
|
-
|
1724
|
-
temp_prefix = '##' if self.flavor != 'oracle' else '_'
|
1725
|
-
temp_target = temp_prefix + transact_id + '_' + pipe.target
|
1722
|
+
temp_target = self.get_temporary_target(pipe.target, label='update')
|
1726
1723
|
self._log_temporary_tables_creation(temp_target, create=(not pipe.temporary), debug=debug)
|
1727
1724
|
temp_pipe = Pipe(
|
1728
1725
|
pipe.connector_keys.replace(':', '_') + '_', pipe.metric_key, pipe.location_key,
|
@@ -1743,7 +1740,7 @@ def sync_pipe(
|
|
1743
1740
|
static=True,
|
1744
1741
|
autoincrement=False,
|
1745
1742
|
parameters={
|
1746
|
-
'schema':
|
1743
|
+
'schema': self.internal_schema,
|
1747
1744
|
'hypertable': False,
|
1748
1745
|
},
|
1749
1746
|
)
|
@@ -1910,15 +1907,18 @@ def sync_pipe_inplace(
|
|
1910
1907
|
)
|
1911
1908
|
from meerschaum.utils.misc import generate_password
|
1912
1909
|
|
1913
|
-
|
1914
|
-
|
1915
|
-
|
1916
|
-
|
1910
|
+
transaction_id_length = (
|
1911
|
+
mrsm.get_config(
|
1912
|
+
'system', 'connectors', 'sql', 'instance', 'temporary_target', 'transaction_id_length'
|
1913
|
+
)
|
1914
|
+
)
|
1915
|
+
transact_id = generate_password(transaction_id_length)
|
1917
1916
|
|
1918
1917
|
internal_schema = self.internal_schema
|
1918
|
+
target = pipe.target
|
1919
1919
|
temp_table_roots = ['backtrack', 'new', 'delta', 'joined', 'unseen', 'update']
|
1920
1920
|
temp_tables = {
|
1921
|
-
table_root:
|
1921
|
+
table_root: self.get_temporary_target(target, transact_id=transact_id, label=table_root)
|
1922
1922
|
for table_root in temp_table_roots
|
1923
1923
|
}
|
1924
1924
|
temp_table_names = {
|
@@ -3064,7 +3064,12 @@ def get_alter_columns_queries(
|
|
3064
3064
|
return []
|
3065
3065
|
if pipe.static:
|
3066
3066
|
return
|
3067
|
-
from meerschaum.utils.sql import
|
3067
|
+
from meerschaum.utils.sql import (
|
3068
|
+
sql_item_name,
|
3069
|
+
get_table_cols_types,
|
3070
|
+
DROP_IF_EXISTS_FLAVORS,
|
3071
|
+
SINGLE_ALTER_TABLE_FLAVORS,
|
3072
|
+
)
|
3068
3073
|
from meerschaum.utils.dataframe import get_numeric_cols
|
3069
3074
|
from meerschaum.utils.dtypes import are_dtypes_equal
|
3070
3075
|
from meerschaum.utils.dtypes.sql import (
|
@@ -3308,14 +3313,19 @@ def get_alter_columns_queries(
|
|
3308
3313
|
else 'TYPE '
|
3309
3314
|
)
|
3310
3315
|
column_str = 'COLUMN' if self.flavor != 'oracle' else ''
|
3311
|
-
|
3316
|
+
query_suffix = (
|
3312
3317
|
f"\n{alter_col_prefix} {column_str} "
|
3313
3318
|
+ sql_item_name(col, self.flavor, None)
|
3314
3319
|
+ " " + type_prefix + typ + ","
|
3315
3320
|
)
|
3321
|
+
if self.flavor not in SINGLE_ALTER_TABLE_FLAVORS:
|
3322
|
+
query += query_suffix
|
3323
|
+
else:
|
3324
|
+
queries.append(query + query_suffix[:-1])
|
3325
|
+
|
3326
|
+
if self.flavor not in SINGLE_ALTER_TABLE_FLAVORS:
|
3327
|
+
queries.append(query[:-1])
|
3316
3328
|
|
3317
|
-
query = query[:-1]
|
3318
|
-
queries.append(query)
|
3319
3329
|
if self.flavor != 'duckdb':
|
3320
3330
|
return queries
|
3321
3331
|
|
@@ -3616,3 +3626,30 @@ def get_pipe_schema(self, pipe: mrsm.Pipe) -> Union[str, None]:
|
|
3616
3626
|
A schema string or `None` if nothing is configured.
|
3617
3627
|
"""
|
3618
3628
|
return pipe.parameters.get('schema', self.schema)
|
3629
|
+
|
3630
|
+
|
3631
|
+
@staticmethod
|
3632
|
+
def get_temporary_target(
|
3633
|
+
target: str,
|
3634
|
+
transact_id: Optional[str, None] = None,
|
3635
|
+
label: Optional[str] = None,
|
3636
|
+
separator: Optional[str] = None,
|
3637
|
+
) -> str:
|
3638
|
+
"""
|
3639
|
+
Return a unique(ish) temporary target for a pipe.
|
3640
|
+
"""
|
3641
|
+
from meerschaum.utils.misc import generate_password
|
3642
|
+
temp_target_cf = (
|
3643
|
+
mrsm.get_config('system', 'connectors', 'sql', 'instance', 'temporary_target') or {}
|
3644
|
+
)
|
3645
|
+
transaction_id_len = temp_target_cf.get('transaction_id_length', 3)
|
3646
|
+
transact_id = transact_id or generate_password(transaction_id_len)
|
3647
|
+
temp_prefix = temp_target_cf.get('prefix', '_')
|
3648
|
+
separator = separator or temp_target_cf.get('separator', '_')
|
3649
|
+
return (
|
3650
|
+
temp_prefix
|
3651
|
+
+ target
|
3652
|
+
+ separator
|
3653
|
+
+ transact_id
|
3654
|
+
+ ((separator + label) if label else '')
|
3655
|
+
)
|