meerschaum 2.6.16__py3-none-any.whl → 2.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/_internal/arguments/_parse_arguments.py +1 -1
- meerschaum/actions/delete.py +65 -69
- meerschaum/actions/edit.py +22 -2
- meerschaum/actions/install.py +1 -2
- meerschaum/actions/sync.py +2 -3
- meerschaum/api/routes/_pipes.py +7 -8
- meerschaum/config/_default.py +1 -1
- meerschaum/config/_paths.py +2 -1
- meerschaum/config/_version.py +1 -1
- meerschaum/connectors/api/_pipes.py +18 -21
- meerschaum/connectors/sql/_create_engine.py +3 -3
- meerschaum/connectors/sql/_instance.py +11 -12
- meerschaum/connectors/sql/_pipes.py +143 -91
- meerschaum/connectors/sql/_sql.py +43 -8
- meerschaum/connectors/valkey/_pipes.py +12 -1
- meerschaum/core/Pipe/__init__.py +23 -13
- meerschaum/core/Pipe/_attributes.py +25 -1
- meerschaum/core/Pipe/_dtypes.py +23 -16
- meerschaum/core/Pipe/_sync.py +59 -31
- meerschaum/core/Pipe/_verify.py +8 -7
- meerschaum/jobs/_Job.py +4 -1
- meerschaum/plugins/_Plugin.py +11 -14
- meerschaum/utils/daemon/Daemon.py +22 -15
- meerschaum/utils/dataframe.py +178 -16
- meerschaum/utils/dtypes/__init__.py +149 -14
- meerschaum/utils/dtypes/sql.py +41 -7
- meerschaum/utils/misc.py +8 -8
- meerschaum/utils/packages/_packages.py +1 -1
- meerschaum/utils/schedule.py +8 -3
- meerschaum/utils/sql.py +180 -100
- meerschaum/utils/venv/_Venv.py +4 -4
- meerschaum/utils/venv/__init__.py +53 -20
- {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/METADATA +2 -2
- {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/RECORD +40 -40
- {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/LICENSE +0 -0
- {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/NOTICE +0 -0
- {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/WHEEL +0 -0
- {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/top_level.txt +0 -0
- {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/zip-safe +0 -0
meerschaum/actions/delete.py
CHANGED
@@ -9,6 +9,7 @@ Functions for deleting elements.
|
|
9
9
|
from __future__ import annotations
|
10
10
|
from meerschaum.utils.typing import Any, SuccessTuple, Union, Optional, List
|
11
11
|
|
12
|
+
|
12
13
|
def delete(
|
13
14
|
action: Optional[List[str]] = None,
|
14
15
|
**kw: Any
|
@@ -21,7 +22,6 @@ def delete(
|
|
21
22
|
|
22
23
|
"""
|
23
24
|
from meerschaum.actions import choose_subaction
|
24
|
-
from meerschaum.utils.debug import dprint
|
25
25
|
options = {
|
26
26
|
'config' : _delete_config,
|
27
27
|
'pipes' : _delete_pipes,
|
@@ -65,19 +65,18 @@ def _complete_delete(
|
|
65
65
|
|
66
66
|
|
67
67
|
def _delete_pipes(
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
68
|
+
debug: bool = False,
|
69
|
+
yes: bool = False,
|
70
|
+
force: bool = False,
|
71
|
+
noask: bool = False,
|
72
|
+
**kw: Any
|
73
|
+
) -> SuccessTuple:
|
74
74
|
"""
|
75
75
|
Drop pipes and delete their registrations.
|
76
|
-
|
77
76
|
"""
|
78
77
|
from meerschaum import get_pipes
|
79
78
|
from meerschaum.utils.prompt import yes_no
|
80
|
-
from meerschaum.utils.formatting import
|
79
|
+
from meerschaum.utils.formatting import highlight_pipes
|
81
80
|
from meerschaum.utils.warnings import warn
|
82
81
|
from meerschaum.actions import actions
|
83
82
|
pipes = get_pipes(as_list=True, debug=debug, **kw)
|
@@ -126,21 +125,22 @@ def _delete_pipes(
|
|
126
125
|
|
127
126
|
return successes > 0, msg
|
128
127
|
|
128
|
+
|
129
129
|
def _delete_config(
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
130
|
+
action: Optional[List[str]] = None,
|
131
|
+
yes: bool = False,
|
132
|
+
force: bool = False,
|
133
|
+
noask: bool = False,
|
134
|
+
debug: bool = False,
|
135
|
+
**kw: Any
|
136
|
+
) -> SuccessTuple:
|
137
137
|
"""
|
138
138
|
Delete configuration files.
|
139
|
-
|
140
139
|
"""
|
141
|
-
import os
|
140
|
+
import os
|
141
|
+
import shutil
|
142
142
|
from meerschaum.utils.prompt import yes_no
|
143
|
-
from meerschaum.config._paths import
|
143
|
+
from meerschaum.config._paths import STACK_COMPOSE_PATH, DEFAULT_CONFIG_DIR_PATH
|
144
144
|
from meerschaum.config._read_config import get_possible_keys, get_keyfile_path
|
145
145
|
from meerschaum.utils.debug import dprint
|
146
146
|
paths = [p for p in [STACK_COMPOSE_PATH, DEFAULT_CONFIG_DIR_PATH] if p.exists()]
|
@@ -157,7 +157,7 @@ def _delete_config(
|
|
157
157
|
else:
|
158
158
|
sep = '\n' + ' - '
|
159
159
|
answer = yes_no(
|
160
|
-
|
160
|
+
"Are you sure you want to delete the following configuration files?" +
|
161
161
|
f"{sep + sep.join([str(p) for p in paths])}\n",
|
162
162
|
default='n', noask=noask, yes=yes
|
163
163
|
)
|
@@ -176,18 +176,18 @@ def _delete_config(
|
|
176
176
|
|
177
177
|
return success, msg
|
178
178
|
|
179
|
+
|
179
180
|
def _delete_plugins(
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
181
|
+
action: Optional[List[str]] = None,
|
182
|
+
repository: Optional[str] = None,
|
183
|
+
yes: bool = False,
|
184
|
+
force: bool = False,
|
185
|
+
noask: bool = False,
|
186
|
+
debug: bool = False,
|
187
|
+
**kw: Any
|
188
|
+
) -> SuccessTuple:
|
188
189
|
"""
|
189
190
|
Delete plugins from a Meerschaum repository.
|
190
|
-
|
191
191
|
"""
|
192
192
|
from meerschaum.utils.warnings import info
|
193
193
|
from meerschaum.plugins import reload_plugins
|
@@ -206,7 +206,7 @@ def _delete_plugins(
|
|
206
206
|
) if not force else True
|
207
207
|
|
208
208
|
if not answer:
|
209
|
-
return False,
|
209
|
+
return False, "No plugins deleted."
|
210
210
|
|
211
211
|
successes = {}
|
212
212
|
for name in action:
|
@@ -218,27 +218,24 @@ def _delete_plugins(
|
|
218
218
|
reload_plugins(debug=debug)
|
219
219
|
return True, "Success"
|
220
220
|
|
221
|
+
|
221
222
|
def _delete_users(
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
223
|
+
action: Optional[List[str]] = None,
|
224
|
+
mrsm_instance: Optional[str] = None,
|
225
|
+
yes: bool = False,
|
226
|
+
force: bool = False,
|
227
|
+
noask: bool = False,
|
228
|
+
shell: bool = False,
|
229
|
+
debug: bool = False,
|
230
|
+
**kw
|
231
|
+
) -> SuccessTuple:
|
231
232
|
"""
|
232
233
|
Delete users from a Meerschaum instance. Adequate permissions are required.
|
233
|
-
|
234
234
|
"""
|
235
|
-
from meerschaum import get_connector
|
236
235
|
from meerschaum.connectors.parse import parse_instance_keys
|
237
|
-
from meerschaum.utils.prompt import yes_no
|
238
|
-
from meerschaum.utils.
|
239
|
-
from meerschaum.utils.warnings import warn, error, info
|
236
|
+
from meerschaum.utils.prompt import yes_no
|
237
|
+
from meerschaum.utils.warnings import info
|
240
238
|
from meerschaum.core import User
|
241
|
-
from meerschaum.connectors.api import APIConnector
|
242
239
|
from meerschaum.utils.formatting import print_tuple
|
243
240
|
instance_connector = parse_instance_keys(mrsm_instance)
|
244
241
|
|
@@ -292,24 +289,25 @@ def _delete_users(
|
|
292
289
|
)
|
293
290
|
return True, msg
|
294
291
|
|
292
|
+
|
295
293
|
def _delete_connectors(
|
296
|
-
|
297
|
-
|
298
|
-
|
299
|
-
|
300
|
-
|
301
|
-
|
302
|
-
|
303
|
-
|
294
|
+
action: Optional[List[str]] = None,
|
295
|
+
connector_keys: Optional[List[str]] = None,
|
296
|
+
yes: bool = False,
|
297
|
+
force: bool = False,
|
298
|
+
noask: bool = False,
|
299
|
+
debug: bool = False,
|
300
|
+
**kw: Any
|
301
|
+
) -> SuccessTuple:
|
304
302
|
"""
|
305
303
|
Delete configured connectors.
|
306
304
|
|
307
305
|
Example:
|
308
306
|
`delete connectors sql:test`
|
309
|
-
|
310
307
|
"""
|
311
|
-
import os
|
312
|
-
|
308
|
+
import os
|
309
|
+
import pathlib
|
310
|
+
from meerschaum.utils.prompt import yes_no
|
313
311
|
from meerschaum.connectors.parse import parse_connector_keys
|
314
312
|
from meerschaum.config import _config
|
315
313
|
from meerschaum.config._edit import write_config
|
@@ -329,7 +327,7 @@ def _delete_connectors(
|
|
329
327
|
for ck in _keys:
|
330
328
|
try:
|
331
329
|
conn = parse_connector_keys(ck, debug=debug)
|
332
|
-
except Exception
|
330
|
+
except Exception:
|
333
331
|
warn(f"Could not parse connector '{ck}'. Skipping...", stack=False)
|
334
332
|
continue
|
335
333
|
|
@@ -357,26 +355,27 @@ def _delete_connectors(
|
|
357
355
|
):
|
358
356
|
try:
|
359
357
|
os.remove(c.database)
|
360
|
-
except Exception
|
358
|
+
except Exception:
|
361
359
|
warn(
|
362
360
|
"Failed to delete database file for connector "
|
363
361
|
+ f"'{c}'. Ignoring...", stack=False
|
364
362
|
)
|
365
|
-
except Exception
|
363
|
+
except Exception:
|
366
364
|
pass
|
367
365
|
try:
|
368
366
|
del cf['meerschaum']['connectors'][c.type][c.label]
|
369
|
-
except Exception
|
367
|
+
except Exception:
|
370
368
|
warn(f"Failed to delete connector '{c}' from configuration. Skipping...", stack=False)
|
371
369
|
|
372
370
|
write_config(cf, debug=debug)
|
373
371
|
return True, "Success"
|
374
372
|
|
373
|
+
|
375
374
|
def _complete_delete_connectors(
|
376
|
-
|
377
|
-
|
378
|
-
|
379
|
-
|
375
|
+
action: Optional[List[str]] = None,
|
376
|
+
line: str = '',
|
377
|
+
**kw: Any
|
378
|
+
) -> List[str]:
|
380
379
|
from meerschaum.config import get_config
|
381
380
|
from meerschaum.utils.misc import get_connector_labels
|
382
381
|
types = list(get_config('meerschaum', 'connectors').keys())
|
@@ -401,10 +400,8 @@ def _delete_jobs(
|
|
401
400
|
Remove a job's log files and delete the job's ID.
|
402
401
|
|
403
402
|
If the job is running, ask to kill the job first.
|
404
|
-
|
405
403
|
"""
|
406
404
|
from meerschaum.jobs import (
|
407
|
-
Job,
|
408
405
|
get_running_jobs,
|
409
406
|
get_stopped_jobs,
|
410
407
|
get_filtered_jobs,
|
@@ -460,7 +457,7 @@ def _delete_jobs(
|
|
460
457
|
### Ensure the running jobs are dead.
|
461
458
|
if get_running_jobs(executor_keys, jobs, debug=debug):
|
462
459
|
return False, (
|
463
|
-
|
460
|
+
"Failed to kill running jobs. Please stop these jobs before deleting."
|
464
461
|
)
|
465
462
|
_to_delete.update(to_stop_jobs)
|
466
463
|
|
@@ -475,7 +472,7 @@ def _delete_jobs(
|
|
475
472
|
pprint_jobs(_to_delete, nopretty=nopretty)
|
476
473
|
if not yes_no(
|
477
474
|
"Are you sure you want to delete these jobs?",
|
478
|
-
yes=yes, noask=noask, default='
|
475
|
+
yes=yes, noask=noask, default='n',
|
479
476
|
):
|
480
477
|
return False, "No jobs were deleted."
|
481
478
|
|
@@ -569,7 +566,6 @@ def _delete_venvs(
|
|
569
566
|
from meerschaum.utils.venv import venv_exists
|
570
567
|
from meerschaum.utils.prompt import yes_no
|
571
568
|
from meerschaum.utils.misc import print_options
|
572
|
-
from meerschaum.utils.warnings import warn
|
573
569
|
|
574
570
|
venvs_to_skip = ['mrsm']
|
575
571
|
venvs = [
|
meerschaum/actions/edit.py
CHANGED
@@ -397,6 +397,7 @@ def _edit_jobs(
|
|
397
397
|
from meerschaum._internal.arguments import (
|
398
398
|
split_pipeline_sysargs,
|
399
399
|
split_chained_sysargs,
|
400
|
+
parse_arguments,
|
400
401
|
)
|
401
402
|
from meerschaum.utils.formatting import make_header, print_options
|
402
403
|
from meerschaum.utils.warnings import info
|
@@ -410,7 +411,26 @@ def _edit_jobs(
|
|
410
411
|
|
411
412
|
num_edited = 0
|
412
413
|
for name, job in jobs.items():
|
413
|
-
|
414
|
+
try:
|
415
|
+
sub_args_line = None
|
416
|
+
pipeline_args_line = None
|
417
|
+
if job.sysargs[:2] == ['start', 'pipeline']:
|
418
|
+
job_args = parse_arguments(job.sysargs)
|
419
|
+
mrsm.pprint(job_args)
|
420
|
+
sub_args_line = job_args['params']['sub_args_line']
|
421
|
+
params_index = job.sysargs[2:].index('-P')
|
422
|
+
indices_to_skip = (params_index, params_index + 1)
|
423
|
+
pipeline_args_line = shlex.join(
|
424
|
+
[a for i, a in enumerate(job.sysargs[2:]) if i not in indices_to_skip]
|
425
|
+
)
|
426
|
+
except (ValueError, IndexError):
|
427
|
+
sub_args_line = None
|
428
|
+
|
429
|
+
sysargs_str = (
|
430
|
+
f"{sub_args_line} : {pipeline_args_line}"
|
431
|
+
if sub_args_line is not None and pipeline_args_line is not None
|
432
|
+
else shlex.join(job.sysargs)
|
433
|
+
)
|
414
434
|
clear_screen(debug=debug)
|
415
435
|
info(
|
416
436
|
f"Editing arguments for job '{name}'.\n"
|
@@ -422,7 +442,7 @@ def _edit_jobs(
|
|
422
442
|
try:
|
423
443
|
new_sysargs_str = prompt(
|
424
444
|
"",
|
425
|
-
default_editable=
|
445
|
+
default_editable=job.label,
|
426
446
|
multiline=True,
|
427
447
|
icon=False,
|
428
448
|
completer=ShellCompleter(),
|
meerschaum/actions/install.py
CHANGED
@@ -175,7 +175,7 @@ def _install_packages(
|
|
175
175
|
+ f" into the virtual environment '{venv}'."
|
176
176
|
)
|
177
177
|
return False, (
|
178
|
-
|
178
|
+
"Failed to install package" + ("s" if len(action) != 1 else '') + f" {items_str(action)}."
|
179
179
|
)
|
180
180
|
|
181
181
|
|
@@ -200,7 +200,6 @@ def _install_required(
|
|
200
200
|
from meerschaum.core import Plugin
|
201
201
|
from meerschaum.utils.warnings import warn, info
|
202
202
|
from meerschaum.connectors.parse import parse_repo_keys
|
203
|
-
from meerschaum.utils.formatting import print_tuple
|
204
203
|
from meerschaum.plugins import get_plugins_names
|
205
204
|
repo_connector = parse_repo_keys(repository)
|
206
205
|
|
meerschaum/actions/sync.py
CHANGED
@@ -282,9 +282,8 @@ def _sync_pipes(
|
|
282
282
|
from meerschaum.utils.formatting import print_pipes_results
|
283
283
|
from meerschaum.config.static import STATIC_CONFIG
|
284
284
|
|
285
|
-
### NOTE: Removed MRSM_NONINTERACTIVE check.
|
286
285
|
noninteractive_val = os.environ.get(STATIC_CONFIG['environment']['noninteractive'], None)
|
287
|
-
|
286
|
+
noninteractive = str(noninteractive_val).lower() in ('1', 'true', 'yes')
|
288
287
|
|
289
288
|
run = True
|
290
289
|
msg = ""
|
@@ -292,7 +291,7 @@ def _sync_pipes(
|
|
292
291
|
cooldown = 2 * (min_seconds + 1)
|
293
292
|
success_pipes, failure_pipes = [], []
|
294
293
|
while run:
|
295
|
-
_progress = progress() if shell else None
|
294
|
+
_progress = progress() if shell and not noninteractive else None
|
296
295
|
cm = _progress if _progress is not None else contextlib.nullcontext()
|
297
296
|
|
298
297
|
lap_begin = time.perf_counter()
|
meerschaum/api/routes/_pipes.py
CHANGED
@@ -11,7 +11,6 @@ from __future__ import annotations
|
|
11
11
|
|
12
12
|
import io
|
13
13
|
import json
|
14
|
-
import fastapi
|
15
14
|
from decimal import Decimal
|
16
15
|
import datetime
|
17
16
|
|
@@ -359,16 +358,16 @@ def sync_pipe(
|
|
359
358
|
p = get_pipe(connector_keys, metric_key, location_key)
|
360
359
|
if p.target in ('users', 'plugins', 'pipes'):
|
361
360
|
raise fastapi.HTTPException(
|
362
|
-
status_code
|
363
|
-
detail
|
361
|
+
status_code=409,
|
362
|
+
detail=f"Cannot sync data to protected table '{p.target}'.",
|
364
363
|
)
|
365
364
|
|
366
365
|
if not p.columns and columns is not None:
|
367
366
|
p.columns = json.loads(columns)
|
368
367
|
if not p.columns and not is_pipe_registered(p, pipes(refresh=True)):
|
369
368
|
raise fastapi.HTTPException(
|
370
|
-
status_code
|
371
|
-
detail
|
369
|
+
status_code=409,
|
370
|
+
detail="Pipe must be registered with index columns specified."
|
372
371
|
)
|
373
372
|
|
374
373
|
result = list(p.sync(
|
@@ -412,7 +411,7 @@ def get_pipe_data(
|
|
412
411
|
if params is not None:
|
413
412
|
try:
|
414
413
|
_params = json.loads(params)
|
415
|
-
except Exception
|
414
|
+
except Exception:
|
416
415
|
_params = None
|
417
416
|
if not isinstance(_params, dict):
|
418
417
|
raise fastapi.HTTPException(
|
@@ -426,7 +425,7 @@ def get_pipe_data(
|
|
426
425
|
if select_columns is not None:
|
427
426
|
try:
|
428
427
|
_select_columns = json.loads(select_columns)
|
429
|
-
except Exception
|
428
|
+
except Exception:
|
430
429
|
_select_columns = None
|
431
430
|
if not isinstance(_select_columns, list):
|
432
431
|
raise fastapi.HTTPException(
|
@@ -440,7 +439,7 @@ def get_pipe_data(
|
|
440
439
|
if omit_columns is not None:
|
441
440
|
try:
|
442
441
|
_omit_columns = json.loads(omit_columns)
|
443
|
-
except Exception
|
442
|
+
except Exception:
|
444
443
|
_omit_columns = None
|
445
444
|
if _omit_columns is None:
|
446
445
|
raise fastapi.HTTPException(
|
meerschaum/config/_default.py
CHANGED
meerschaum/config/_paths.py
CHANGED
@@ -103,7 +103,7 @@ if ENVIRONMENT_VENVS_DIR in os.environ:
|
|
103
103
|
if not _VENVS_DIR_PATH.exists():
|
104
104
|
try:
|
105
105
|
_VENVS_DIR_PATH.mkdir(parents=True, exist_ok=True)
|
106
|
-
except Exception
|
106
|
+
except Exception:
|
107
107
|
print(
|
108
108
|
f"Invalid path set for environment variable '{ENVIRONMENT_VENVS_DIR}':\n"
|
109
109
|
+ f"{_VENVS_DIR_PATH}"
|
@@ -148,6 +148,7 @@ paths = {
|
|
148
148
|
'CACHE_RESOURCES_PATH' : ('{ROOT_DIR_PATH}', '.cache'),
|
149
149
|
'PIPES_CACHE_RESOURCES_PATH' : ('{CACHE_RESOURCES_PATH}', 'pipes'),
|
150
150
|
'USERS_CACHE_RESOURCES_PATH' : ('{CACHE_RESOURCES_PATH}', 'users'),
|
151
|
+
'VENVS_CACHE_RESOURCES_PATH' : ('{CACHE_RESOURCES_PATH}', 'venvs'),
|
151
152
|
|
152
153
|
'PLUGINS_RESOURCES_PATH' : ('{INTERNAL_RESOURCES_PATH}', 'plugins'),
|
153
154
|
'PLUGINS_INTERNAL_LOCK_PATH' : ('{INTERNAL_RESOURCES_PATH}', 'plugins.lock'),
|
meerschaum/config/_version.py
CHANGED
@@ -15,7 +15,8 @@ from datetime import datetime
|
|
15
15
|
import meerschaum as mrsm
|
16
16
|
from meerschaum.utils.debug import dprint
|
17
17
|
from meerschaum.utils.warnings import warn, error
|
18
|
-
from meerschaum.utils.typing import SuccessTuple, Union, Any, Optional,
|
18
|
+
from meerschaum.utils.typing import SuccessTuple, Union, Any, Optional, List, Dict, Tuple
|
19
|
+
|
19
20
|
|
20
21
|
def pipe_r_url(
|
21
22
|
pipe: mrsm.Pipe
|
@@ -30,6 +31,7 @@ def pipe_r_url(
|
|
30
31
|
+ f"{pipe.connector_keys}/{pipe.metric_key}/{location_key}"
|
31
32
|
)
|
32
33
|
|
34
|
+
|
33
35
|
def register_pipe(
|
34
36
|
self,
|
35
37
|
pipe: mrsm.Pipe,
|
@@ -39,7 +41,6 @@ def register_pipe(
|
|
39
41
|
Returns a tuple of (success_bool, response_dict).
|
40
42
|
"""
|
41
43
|
from meerschaum.utils.debug import dprint
|
42
|
-
from meerschaum.config.static import STATIC_CONFIG
|
43
44
|
### NOTE: if `parameters` is supplied in the Pipe constructor,
|
44
45
|
### then `pipe.parameters` will exist and not be fetched from the database.
|
45
46
|
r_url = pipe_r_url(pipe)
|
@@ -180,7 +181,7 @@ def sync_pipe(
|
|
180
181
|
from meerschaum.utils.misc import json_serialize_datetime, items_str
|
181
182
|
from meerschaum.config import get_config
|
182
183
|
from meerschaum.utils.packages import attempt_import
|
183
|
-
from meerschaum.utils.dataframe import get_numeric_cols, to_json
|
184
|
+
from meerschaum.utils.dataframe import get_numeric_cols, to_json, get_bytes_cols
|
184
185
|
begin = time.time()
|
185
186
|
more_itertools = attempt_import('more_itertools')
|
186
187
|
if df is None:
|
@@ -291,7 +292,7 @@ def sync_pipe(
|
|
291
292
|
|
292
293
|
try:
|
293
294
|
j = tuple(j)
|
294
|
-
except Exception
|
295
|
+
except Exception:
|
295
296
|
return False, response.text
|
296
297
|
|
297
298
|
if debug:
|
@@ -313,12 +314,12 @@ def sync_pipe(
|
|
313
314
|
|
314
315
|
def delete_pipe(
|
315
316
|
self,
|
316
|
-
pipe: Optional[
|
317
|
+
pipe: Optional[mrsm.Pipe] = None,
|
317
318
|
debug: bool = None,
|
318
319
|
) -> SuccessTuple:
|
319
320
|
"""Delete a Pipe and drop its table."""
|
320
321
|
if pipe is None:
|
321
|
-
error(
|
322
|
+
error("Pipe cannot be None.")
|
322
323
|
r_url = pipe_r_url(pipe)
|
323
324
|
response = self.delete(
|
324
325
|
r_url + '/delete',
|
@@ -339,7 +340,7 @@ def delete_pipe(
|
|
339
340
|
|
340
341
|
def get_pipe_data(
|
341
342
|
self,
|
342
|
-
pipe:
|
343
|
+
pipe: mrsm.Pipe,
|
343
344
|
select_columns: Optional[List[str]] = None,
|
344
345
|
omit_columns: Optional[List[str]] = None,
|
345
346
|
begin: Union[str, datetime, int, None] = None,
|
@@ -351,7 +352,6 @@ def get_pipe_data(
|
|
351
352
|
) -> Union[pandas.DataFrame, None]:
|
352
353
|
"""Fetch data from the API."""
|
353
354
|
r_url = pipe_r_url(pipe)
|
354
|
-
chunks_list = []
|
355
355
|
while True:
|
356
356
|
try:
|
357
357
|
response = self.get(
|
@@ -375,12 +375,19 @@ def get_pipe_data(
|
|
375
375
|
return False, j['detail']
|
376
376
|
break
|
377
377
|
|
378
|
-
from meerschaum.utils.packages import import_pandas
|
379
378
|
from meerschaum.utils.dataframe import parse_df_datetimes, add_missing_cols_to_df
|
380
379
|
from meerschaum.utils.dtypes import are_dtypes_equal
|
381
|
-
pd = import_pandas()
|
382
380
|
try:
|
383
|
-
df =
|
381
|
+
df = parse_df_datetimes(
|
382
|
+
j,
|
383
|
+
ignore_cols=[
|
384
|
+
col
|
385
|
+
for col, dtype in pipe.dtypes.items()
|
386
|
+
if not are_dtypes_equal(str(dtype), 'datetime')
|
387
|
+
],
|
388
|
+
strip_timezone=(pipe.tzinfo is None),
|
389
|
+
debug=debug,
|
390
|
+
)
|
384
391
|
except Exception as e:
|
385
392
|
warn(f"Failed to parse response for {pipe}:\n{e}")
|
386
393
|
return None
|
@@ -388,16 +395,6 @@ def get_pipe_data(
|
|
388
395
|
if len(df.columns) == 0:
|
389
396
|
return add_missing_cols_to_df(df, pipe.dtypes)
|
390
397
|
|
391
|
-
df = parse_df_datetimes(
|
392
|
-
df,
|
393
|
-
ignore_cols = [
|
394
|
-
col
|
395
|
-
for col, dtype in pipe.dtypes.items()
|
396
|
-
if not are_dtypes_equal(str(dtype), 'datetime')
|
397
|
-
],
|
398
|
-
strip_timezone=(pipe.tzinfo is None),
|
399
|
-
debug=debug,
|
400
|
-
)
|
401
398
|
return df
|
402
399
|
|
403
400
|
|
@@ -107,7 +107,7 @@ flavor_configs = {
|
|
107
107
|
},
|
108
108
|
},
|
109
109
|
'oracle': {
|
110
|
-
'engine': 'oracle+
|
110
|
+
'engine': 'oracle+oracledb',
|
111
111
|
'create_engine': default_create_engine_args,
|
112
112
|
'omit_create_engine': {'method',},
|
113
113
|
'to_sql': {
|
@@ -164,7 +164,7 @@ install_flavor_drivers = {
|
|
164
164
|
'citus': ['psycopg'],
|
165
165
|
'cockroachdb': ['psycopg', 'sqlalchemy_cockroachdb', 'sqlalchemy_cockroachdb.psycopg'],
|
166
166
|
'mssql': ['pyodbc'],
|
167
|
-
'oracle': ['
|
167
|
+
'oracle': ['oracledb'],
|
168
168
|
}
|
169
169
|
require_patching_flavors = {'cockroachdb': [('sqlalchemy-cockroachdb', 'sqlalchemy_cockroachdb')]}
|
170
170
|
|
@@ -239,7 +239,7 @@ def create_engine(
|
|
239
239
|
self._sys_config['create_engine'] = {}
|
240
240
|
if 'connect_args' not in self._sys_config['create_engine']:
|
241
241
|
self._sys_config['create_engine']['connect_args'] = {}
|
242
|
-
self._sys_config['create_engine']['connect_args'].update({"check_same_thread"
|
242
|
+
self._sys_config['create_engine']['connect_args'].update({"check_same_thread": False})
|
243
243
|
else:
|
244
244
|
engine_str = (
|
245
245
|
_engine + "://" + (_username if _username is not None else '') +
|
@@ -9,8 +9,7 @@ Define utilities for instance connectors.
|
|
9
9
|
import time
|
10
10
|
from datetime import datetime, timezone, timedelta
|
11
11
|
import meerschaum as mrsm
|
12
|
-
from meerschaum.utils.typing import Dict, SuccessTuple,
|
13
|
-
from meerschaum.utils.warnings import warn
|
12
|
+
from meerschaum.utils.typing import Dict, SuccessTuple, Union, List
|
14
13
|
|
15
14
|
|
16
15
|
_in_memory_temp_tables: Dict[str, bool] = {}
|
@@ -28,9 +27,9 @@ def _log_temporary_tables_creation(
|
|
28
27
|
from meerschaum.connectors.sql.tables import get_tables
|
29
28
|
sqlalchemy = mrsm.attempt_import('sqlalchemy')
|
30
29
|
temp_tables_table = get_tables(
|
31
|
-
mrsm_instance
|
32
|
-
create
|
33
|
-
debug
|
30
|
+
mrsm_instance=self,
|
31
|
+
create=create,
|
32
|
+
debug=debug,
|
34
33
|
)['temp_tables']
|
35
34
|
if isinstance(tables, str):
|
36
35
|
tables = [tables]
|
@@ -72,7 +71,9 @@ def _drop_temporary_table(
|
|
72
71
|
return True, "Success"
|
73
72
|
|
74
73
|
drop_query = f"DROP TABLE {if_exists} " + sql_item_name(
|
75
|
-
table,
|
74
|
+
table,
|
75
|
+
self.flavor,
|
76
|
+
schema=self.internal_schema
|
76
77
|
)
|
77
78
|
drop_success = self.exec(drop_query, silent=True, debug=debug) is not None
|
78
79
|
drop_msg = "Success" if drop_success else f"Failed to drop temporary table '{table}'."
|
@@ -83,7 +84,6 @@ def _drop_temporary_tables(self, debug: bool = False) -> SuccessTuple:
|
|
83
84
|
"""
|
84
85
|
Drop all tables in the internal schema that are marked as ready to be dropped.
|
85
86
|
"""
|
86
|
-
from meerschaum.utils.sql import sql_item_name, table_exists
|
87
87
|
from meerschaum.utils.misc import items_str
|
88
88
|
from meerschaum.connectors.sql.tables import get_tables
|
89
89
|
sqlalchemy = mrsm.attempt_import('sqlalchemy')
|
@@ -141,16 +141,15 @@ def _drop_temporary_tables(self, debug: bool = False) -> SuccessTuple:
|
|
141
141
|
|
142
142
|
|
143
143
|
def _drop_old_temporary_tables(
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
144
|
+
self,
|
145
|
+
refresh: bool = True,
|
146
|
+
debug: bool = False,
|
147
|
+
) -> SuccessTuple:
|
148
148
|
"""
|
149
149
|
Drop temporary tables older than the configured interval (24 hours by default).
|
150
150
|
"""
|
151
151
|
from meerschaum.config import get_config
|
152
152
|
from meerschaum.connectors.sql.tables import get_tables
|
153
|
-
from meerschaum.utils.misc import items_str
|
154
153
|
sqlalchemy = mrsm.attempt_import('sqlalchemy')
|
155
154
|
temp_tables_table = get_tables(mrsm_instance=self, create=False, debug=debug)['temp_tables']
|
156
155
|
last_check = getattr(self, '_stale_temporary_tables_check_timestamp', 0)
|