meerschaum 2.6.16__py3-none-any.whl → 2.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. meerschaum/_internal/arguments/_parse_arguments.py +1 -1
  2. meerschaum/actions/delete.py +65 -69
  3. meerschaum/actions/edit.py +22 -2
  4. meerschaum/actions/install.py +1 -2
  5. meerschaum/actions/sync.py +2 -3
  6. meerschaum/api/routes/_pipes.py +7 -8
  7. meerschaum/config/_default.py +1 -1
  8. meerschaum/config/_paths.py +2 -1
  9. meerschaum/config/_version.py +1 -1
  10. meerschaum/connectors/api/_pipes.py +18 -21
  11. meerschaum/connectors/sql/_create_engine.py +3 -3
  12. meerschaum/connectors/sql/_instance.py +11 -12
  13. meerschaum/connectors/sql/_pipes.py +143 -91
  14. meerschaum/connectors/sql/_sql.py +43 -8
  15. meerschaum/connectors/valkey/_pipes.py +12 -1
  16. meerschaum/core/Pipe/__init__.py +23 -13
  17. meerschaum/core/Pipe/_attributes.py +25 -1
  18. meerschaum/core/Pipe/_dtypes.py +23 -16
  19. meerschaum/core/Pipe/_sync.py +59 -31
  20. meerschaum/core/Pipe/_verify.py +8 -7
  21. meerschaum/jobs/_Job.py +4 -1
  22. meerschaum/plugins/_Plugin.py +11 -14
  23. meerschaum/utils/daemon/Daemon.py +22 -15
  24. meerschaum/utils/dataframe.py +178 -16
  25. meerschaum/utils/dtypes/__init__.py +149 -14
  26. meerschaum/utils/dtypes/sql.py +41 -7
  27. meerschaum/utils/misc.py +8 -8
  28. meerschaum/utils/packages/_packages.py +1 -1
  29. meerschaum/utils/schedule.py +8 -3
  30. meerschaum/utils/sql.py +180 -100
  31. meerschaum/utils/venv/_Venv.py +4 -4
  32. meerschaum/utils/venv/__init__.py +53 -20
  33. {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/METADATA +2 -2
  34. {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/RECORD +40 -40
  35. {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/LICENSE +0 -0
  36. {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/NOTICE +0 -0
  37. {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/WHEEL +0 -0
  38. {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/entry_points.txt +0 -0
  39. {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/top_level.txt +0 -0
  40. {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/zip-safe +0 -0
@@ -226,7 +226,7 @@ def parse_line(line: str) -> Dict[str, Any]:
226
226
  import shlex
227
227
  try:
228
228
  return parse_arguments(shlex.split(line))
229
- except Exception as e:
229
+ except Exception:
230
230
  return {'action': [], 'text': line}
231
231
 
232
232
 
@@ -9,6 +9,7 @@ Functions for deleting elements.
9
9
  from __future__ import annotations
10
10
  from meerschaum.utils.typing import Any, SuccessTuple, Union, Optional, List
11
11
 
12
+
12
13
  def delete(
13
14
  action: Optional[List[str]] = None,
14
15
  **kw: Any
@@ -21,7 +22,6 @@ def delete(
21
22
 
22
23
  """
23
24
  from meerschaum.actions import choose_subaction
24
- from meerschaum.utils.debug import dprint
25
25
  options = {
26
26
  'config' : _delete_config,
27
27
  'pipes' : _delete_pipes,
@@ -65,19 +65,18 @@ def _complete_delete(
65
65
 
66
66
 
67
67
  def _delete_pipes(
68
- debug: bool = False,
69
- yes: bool = False,
70
- force: bool = False,
71
- noask: bool = False,
72
- **kw: Any
73
- ) -> SuccessTuple:
68
+ debug: bool = False,
69
+ yes: bool = False,
70
+ force: bool = False,
71
+ noask: bool = False,
72
+ **kw: Any
73
+ ) -> SuccessTuple:
74
74
  """
75
75
  Drop pipes and delete their registrations.
76
-
77
76
  """
78
77
  from meerschaum import get_pipes
79
78
  from meerschaum.utils.prompt import yes_no
80
- from meerschaum.utils.formatting import pprint, highlight_pipes
79
+ from meerschaum.utils.formatting import highlight_pipes
81
80
  from meerschaum.utils.warnings import warn
82
81
  from meerschaum.actions import actions
83
82
  pipes = get_pipes(as_list=True, debug=debug, **kw)
@@ -126,21 +125,22 @@ def _delete_pipes(
126
125
 
127
126
  return successes > 0, msg
128
127
 
128
+
129
129
  def _delete_config(
130
- action: Optional[List[str]] = None,
131
- yes: bool = False,
132
- force: bool = False,
133
- noask: bool = False,
134
- debug: bool = False,
135
- **kw: Any
136
- ) -> SuccessTuple:
130
+ action: Optional[List[str]] = None,
131
+ yes: bool = False,
132
+ force: bool = False,
133
+ noask: bool = False,
134
+ debug: bool = False,
135
+ **kw: Any
136
+ ) -> SuccessTuple:
137
137
  """
138
138
  Delete configuration files.
139
-
140
139
  """
141
- import os, shutil
140
+ import os
141
+ import shutil
142
142
  from meerschaum.utils.prompt import yes_no
143
- from meerschaum.config._paths import CONFIG_DIR_PATH, STACK_COMPOSE_PATH, DEFAULT_CONFIG_DIR_PATH
143
+ from meerschaum.config._paths import STACK_COMPOSE_PATH, DEFAULT_CONFIG_DIR_PATH
144
144
  from meerschaum.config._read_config import get_possible_keys, get_keyfile_path
145
145
  from meerschaum.utils.debug import dprint
146
146
  paths = [p for p in [STACK_COMPOSE_PATH, DEFAULT_CONFIG_DIR_PATH] if p.exists()]
@@ -157,7 +157,7 @@ def _delete_config(
157
157
  else:
158
158
  sep = '\n' + ' - '
159
159
  answer = yes_no(
160
- f"Are you sure you want to delete the following configuration files?" +
160
+ "Are you sure you want to delete the following configuration files?" +
161
161
  f"{sep + sep.join([str(p) for p in paths])}\n",
162
162
  default='n', noask=noask, yes=yes
163
163
  )
@@ -176,18 +176,18 @@ def _delete_config(
176
176
 
177
177
  return success, msg
178
178
 
179
+
179
180
  def _delete_plugins(
180
- action: Optional[List[str]] = None,
181
- repository: Optional[str] = None,
182
- yes: bool = False,
183
- force: bool = False,
184
- noask: bool = False,
185
- debug: bool = False,
186
- **kw: Any
187
- ) -> SuccessTuple:
181
+ action: Optional[List[str]] = None,
182
+ repository: Optional[str] = None,
183
+ yes: bool = False,
184
+ force: bool = False,
185
+ noask: bool = False,
186
+ debug: bool = False,
187
+ **kw: Any
188
+ ) -> SuccessTuple:
188
189
  """
189
190
  Delete plugins from a Meerschaum repository.
190
-
191
191
  """
192
192
  from meerschaum.utils.warnings import info
193
193
  from meerschaum.plugins import reload_plugins
@@ -206,7 +206,7 @@ def _delete_plugins(
206
206
  ) if not force else True
207
207
 
208
208
  if not answer:
209
- return False, f"No plugins deleted."
209
+ return False, "No plugins deleted."
210
210
 
211
211
  successes = {}
212
212
  for name in action:
@@ -218,27 +218,24 @@ def _delete_plugins(
218
218
  reload_plugins(debug=debug)
219
219
  return True, "Success"
220
220
 
221
+
221
222
  def _delete_users(
222
- action: Optional[List[str]] = None,
223
- mrsm_instance: Optional[str] = None,
224
- yes: bool = False,
225
- force: bool = False,
226
- noask: bool = False,
227
- shell: bool = False,
228
- debug: bool = False,
229
- **kw
230
- ) -> SuccessTuple:
223
+ action: Optional[List[str]] = None,
224
+ mrsm_instance: Optional[str] = None,
225
+ yes: bool = False,
226
+ force: bool = False,
227
+ noask: bool = False,
228
+ shell: bool = False,
229
+ debug: bool = False,
230
+ **kw
231
+ ) -> SuccessTuple:
231
232
  """
232
233
  Delete users from a Meerschaum instance. Adequate permissions are required.
233
-
234
234
  """
235
- from meerschaum import get_connector
236
235
  from meerschaum.connectors.parse import parse_instance_keys
237
- from meerschaum.utils.prompt import yes_no, prompt
238
- from meerschaum.utils.debug import dprint
239
- from meerschaum.utils.warnings import warn, error, info
236
+ from meerschaum.utils.prompt import yes_no
237
+ from meerschaum.utils.warnings import info
240
238
  from meerschaum.core import User
241
- from meerschaum.connectors.api import APIConnector
242
239
  from meerschaum.utils.formatting import print_tuple
243
240
  instance_connector = parse_instance_keys(mrsm_instance)
244
241
 
@@ -292,24 +289,25 @@ def _delete_users(
292
289
  )
293
290
  return True, msg
294
291
 
292
+
295
293
  def _delete_connectors(
296
- action: Optional[List[str]] = None,
297
- connector_keys: Optional[List[str]] = None,
298
- yes: bool = False,
299
- force: bool = False,
300
- noask: bool = False,
301
- debug: bool = False,
302
- **kw: Any
303
- ) -> SuccessTuple:
294
+ action: Optional[List[str]] = None,
295
+ connector_keys: Optional[List[str]] = None,
296
+ yes: bool = False,
297
+ force: bool = False,
298
+ noask: bool = False,
299
+ debug: bool = False,
300
+ **kw: Any
301
+ ) -> SuccessTuple:
304
302
  """
305
303
  Delete configured connectors.
306
304
 
307
305
  Example:
308
306
  `delete connectors sql:test`
309
-
310
307
  """
311
- import os, pathlib
312
- from meerschaum.utils.prompt import yes_no, prompt
308
+ import os
309
+ import pathlib
310
+ from meerschaum.utils.prompt import yes_no
313
311
  from meerschaum.connectors.parse import parse_connector_keys
314
312
  from meerschaum.config import _config
315
313
  from meerschaum.config._edit import write_config
@@ -329,7 +327,7 @@ def _delete_connectors(
329
327
  for ck in _keys:
330
328
  try:
331
329
  conn = parse_connector_keys(ck, debug=debug)
332
- except Exception as e:
330
+ except Exception:
333
331
  warn(f"Could not parse connector '{ck}'. Skipping...", stack=False)
334
332
  continue
335
333
 
@@ -357,26 +355,27 @@ def _delete_connectors(
357
355
  ):
358
356
  try:
359
357
  os.remove(c.database)
360
- except Exception as e:
358
+ except Exception:
361
359
  warn(
362
360
  "Failed to delete database file for connector "
363
361
  + f"'{c}'. Ignoring...", stack=False
364
362
  )
365
- except Exception as e:
363
+ except Exception:
366
364
  pass
367
365
  try:
368
366
  del cf['meerschaum']['connectors'][c.type][c.label]
369
- except Exception as e:
367
+ except Exception:
370
368
  warn(f"Failed to delete connector '{c}' from configuration. Skipping...", stack=False)
371
369
 
372
370
  write_config(cf, debug=debug)
373
371
  return True, "Success"
374
372
 
373
+
375
374
  def _complete_delete_connectors(
376
- action: Optional[List[str]] = None,
377
- line: str = '',
378
- **kw: Any
379
- ) -> List[str]:
375
+ action: Optional[List[str]] = None,
376
+ line: str = '',
377
+ **kw: Any
378
+ ) -> List[str]:
380
379
  from meerschaum.config import get_config
381
380
  from meerschaum.utils.misc import get_connector_labels
382
381
  types = list(get_config('meerschaum', 'connectors').keys())
@@ -401,10 +400,8 @@ def _delete_jobs(
401
400
  Remove a job's log files and delete the job's ID.
402
401
 
403
402
  If the job is running, ask to kill the job first.
404
-
405
403
  """
406
404
  from meerschaum.jobs import (
407
- Job,
408
405
  get_running_jobs,
409
406
  get_stopped_jobs,
410
407
  get_filtered_jobs,
@@ -460,7 +457,7 @@ def _delete_jobs(
460
457
  ### Ensure the running jobs are dead.
461
458
  if get_running_jobs(executor_keys, jobs, debug=debug):
462
459
  return False, (
463
- f"Failed to kill running jobs. Please stop these jobs before deleting."
460
+ "Failed to kill running jobs. Please stop these jobs before deleting."
464
461
  )
465
462
  _to_delete.update(to_stop_jobs)
466
463
 
@@ -475,7 +472,7 @@ def _delete_jobs(
475
472
  pprint_jobs(_to_delete, nopretty=nopretty)
476
473
  if not yes_no(
477
474
  "Are you sure you want to delete these jobs?",
478
- yes=yes, noask=noask, default='y',
475
+ yes=yes, noask=noask, default='n',
479
476
  ):
480
477
  return False, "No jobs were deleted."
481
478
 
@@ -569,7 +566,6 @@ def _delete_venvs(
569
566
  from meerschaum.utils.venv import venv_exists
570
567
  from meerschaum.utils.prompt import yes_no
571
568
  from meerschaum.utils.misc import print_options
572
- from meerschaum.utils.warnings import warn
573
569
 
574
570
  venvs_to_skip = ['mrsm']
575
571
  venvs = [
@@ -397,6 +397,7 @@ def _edit_jobs(
397
397
  from meerschaum._internal.arguments import (
398
398
  split_pipeline_sysargs,
399
399
  split_chained_sysargs,
400
+ parse_arguments,
400
401
  )
401
402
  from meerschaum.utils.formatting import make_header, print_options
402
403
  from meerschaum.utils.warnings import info
@@ -410,7 +411,26 @@ def _edit_jobs(
410
411
 
411
412
  num_edited = 0
412
413
  for name, job in jobs.items():
413
- sysargs_str = shlex.join(job.sysargs)
414
+ try:
415
+ sub_args_line = None
416
+ pipeline_args_line = None
417
+ if job.sysargs[:2] == ['start', 'pipeline']:
418
+ job_args = parse_arguments(job.sysargs)
419
+ mrsm.pprint(job_args)
420
+ sub_args_line = job_args['params']['sub_args_line']
421
+ params_index = job.sysargs[2:].index('-P')
422
+ indices_to_skip = (params_index, params_index + 1)
423
+ pipeline_args_line = shlex.join(
424
+ [a for i, a in enumerate(job.sysargs[2:]) if i not in indices_to_skip]
425
+ )
426
+ except (ValueError, IndexError):
427
+ sub_args_line = None
428
+
429
+ sysargs_str = (
430
+ f"{sub_args_line} : {pipeline_args_line}"
431
+ if sub_args_line is not None and pipeline_args_line is not None
432
+ else shlex.join(job.sysargs)
433
+ )
414
434
  clear_screen(debug=debug)
415
435
  info(
416
436
  f"Editing arguments for job '{name}'.\n"
@@ -422,7 +442,7 @@ def _edit_jobs(
422
442
  try:
423
443
  new_sysargs_str = prompt(
424
444
  "",
425
- default_editable=sysargs_str.lstrip().rstrip().replace(' + ', '\n+ '),
445
+ default_editable=job.label,
426
446
  multiline=True,
427
447
  icon=False,
428
448
  completer=ShellCompleter(),
@@ -175,7 +175,7 @@ def _install_packages(
175
175
  + f" into the virtual environment '{venv}'."
176
176
  )
177
177
  return False, (
178
- f"Failed to install package" + ("s" if len(action) != 1 else '') + f" {items_str(action)}."
178
+ "Failed to install package" + ("s" if len(action) != 1 else '') + f" {items_str(action)}."
179
179
  )
180
180
 
181
181
 
@@ -200,7 +200,6 @@ def _install_required(
200
200
  from meerschaum.core import Plugin
201
201
  from meerschaum.utils.warnings import warn, info
202
202
  from meerschaum.connectors.parse import parse_repo_keys
203
- from meerschaum.utils.formatting import print_tuple
204
203
  from meerschaum.plugins import get_plugins_names
205
204
  repo_connector = parse_repo_keys(repository)
206
205
 
@@ -282,9 +282,8 @@ def _sync_pipes(
282
282
  from meerschaum.utils.formatting import print_pipes_results
283
283
  from meerschaum.config.static import STATIC_CONFIG
284
284
 
285
- ### NOTE: Removed MRSM_NONINTERACTIVE check.
286
285
  noninteractive_val = os.environ.get(STATIC_CONFIG['environment']['noninteractive'], None)
287
- _ = noninteractive_val in ('1', 'true', 'True', 'yes')
286
+ noninteractive = str(noninteractive_val).lower() in ('1', 'true', 'yes')
288
287
 
289
288
  run = True
290
289
  msg = ""
@@ -292,7 +291,7 @@ def _sync_pipes(
292
291
  cooldown = 2 * (min_seconds + 1)
293
292
  success_pipes, failure_pipes = [], []
294
293
  while run:
295
- _progress = progress() if shell else None
294
+ _progress = progress() if shell and not noninteractive else None
296
295
  cm = _progress if _progress is not None else contextlib.nullcontext()
297
296
 
298
297
  lap_begin = time.perf_counter()
@@ -11,7 +11,6 @@ from __future__ import annotations
11
11
 
12
12
  import io
13
13
  import json
14
- import fastapi
15
14
  from decimal import Decimal
16
15
  import datetime
17
16
 
@@ -359,16 +358,16 @@ def sync_pipe(
359
358
  p = get_pipe(connector_keys, metric_key, location_key)
360
359
  if p.target in ('users', 'plugins', 'pipes'):
361
360
  raise fastapi.HTTPException(
362
- status_code = 409,
363
- detail = f"Cannot sync data to protected table '{p.target}'.",
361
+ status_code=409,
362
+ detail=f"Cannot sync data to protected table '{p.target}'.",
364
363
  )
365
364
 
366
365
  if not p.columns and columns is not None:
367
366
  p.columns = json.loads(columns)
368
367
  if not p.columns and not is_pipe_registered(p, pipes(refresh=True)):
369
368
  raise fastapi.HTTPException(
370
- status_code = 409,
371
- detail = "Pipe must be registered with the datetime column specified."
369
+ status_code=409,
370
+ detail="Pipe must be registered with index columns specified."
372
371
  )
373
372
 
374
373
  result = list(p.sync(
@@ -412,7 +411,7 @@ def get_pipe_data(
412
411
  if params is not None:
413
412
  try:
414
413
  _params = json.loads(params)
415
- except Exception as e:
414
+ except Exception:
416
415
  _params = None
417
416
  if not isinstance(_params, dict):
418
417
  raise fastapi.HTTPException(
@@ -426,7 +425,7 @@ def get_pipe_data(
426
425
  if select_columns is not None:
427
426
  try:
428
427
  _select_columns = json.loads(select_columns)
429
- except Exception as e:
428
+ except Exception:
430
429
  _select_columns = None
431
430
  if not isinstance(_select_columns, list):
432
431
  raise fastapi.HTTPException(
@@ -440,7 +439,7 @@ def get_pipe_data(
440
439
  if omit_columns is not None:
441
440
  try:
442
441
  _omit_columns = json.loads(omit_columns)
443
- except Exception as e:
442
+ except Exception:
444
443
  _omit_columns = None
445
444
  if _omit_columns is None:
446
445
  raise fastapi.HTTPException(
@@ -137,7 +137,7 @@ default_pipes_config = {
137
137
  },
138
138
  'verify': {
139
139
  'bound_days': 366,
140
- 'chunk_minutes': 1440,
140
+ 'chunk_minutes': 43200,
141
141
  },
142
142
  },
143
143
  'attributes': {
@@ -103,7 +103,7 @@ if ENVIRONMENT_VENVS_DIR in os.environ:
103
103
  if not _VENVS_DIR_PATH.exists():
104
104
  try:
105
105
  _VENVS_DIR_PATH.mkdir(parents=True, exist_ok=True)
106
- except Exception as e:
106
+ except Exception:
107
107
  print(
108
108
  f"Invalid path set for environment variable '{ENVIRONMENT_VENVS_DIR}':\n"
109
109
  + f"{_VENVS_DIR_PATH}"
@@ -148,6 +148,7 @@ paths = {
148
148
  'CACHE_RESOURCES_PATH' : ('{ROOT_DIR_PATH}', '.cache'),
149
149
  'PIPES_CACHE_RESOURCES_PATH' : ('{CACHE_RESOURCES_PATH}', 'pipes'),
150
150
  'USERS_CACHE_RESOURCES_PATH' : ('{CACHE_RESOURCES_PATH}', 'users'),
151
+ 'VENVS_CACHE_RESOURCES_PATH' : ('{CACHE_RESOURCES_PATH}', 'venvs'),
151
152
 
152
153
  'PLUGINS_RESOURCES_PATH' : ('{INTERNAL_RESOURCES_PATH}', 'plugins'),
153
154
  'PLUGINS_INTERNAL_LOCK_PATH' : ('{INTERNAL_RESOURCES_PATH}', 'plugins.lock'),
@@ -2,4 +2,4 @@
2
2
  Specify the Meerschaum release version.
3
3
  """
4
4
 
5
- __version__ = "2.6.16"
5
+ __version__ = "2.7.0"
@@ -15,7 +15,8 @@ from datetime import datetime
15
15
  import meerschaum as mrsm
16
16
  from meerschaum.utils.debug import dprint
17
17
  from meerschaum.utils.warnings import warn, error
18
- from meerschaum.utils.typing import SuccessTuple, Union, Any, Optional, Mapping, List, Dict, Tuple
18
+ from meerschaum.utils.typing import SuccessTuple, Union, Any, Optional, List, Dict, Tuple
19
+
19
20
 
20
21
  def pipe_r_url(
21
22
  pipe: mrsm.Pipe
@@ -30,6 +31,7 @@ def pipe_r_url(
30
31
  + f"{pipe.connector_keys}/{pipe.metric_key}/{location_key}"
31
32
  )
32
33
 
34
+
33
35
  def register_pipe(
34
36
  self,
35
37
  pipe: mrsm.Pipe,
@@ -39,7 +41,6 @@ def register_pipe(
39
41
  Returns a tuple of (success_bool, response_dict).
40
42
  """
41
43
  from meerschaum.utils.debug import dprint
42
- from meerschaum.config.static import STATIC_CONFIG
43
44
  ### NOTE: if `parameters` is supplied in the Pipe constructor,
44
45
  ### then `pipe.parameters` will exist and not be fetched from the database.
45
46
  r_url = pipe_r_url(pipe)
@@ -180,7 +181,7 @@ def sync_pipe(
180
181
  from meerschaum.utils.misc import json_serialize_datetime, items_str
181
182
  from meerschaum.config import get_config
182
183
  from meerschaum.utils.packages import attempt_import
183
- from meerschaum.utils.dataframe import get_numeric_cols, to_json
184
+ from meerschaum.utils.dataframe import get_numeric_cols, to_json, get_bytes_cols
184
185
  begin = time.time()
185
186
  more_itertools = attempt_import('more_itertools')
186
187
  if df is None:
@@ -291,7 +292,7 @@ def sync_pipe(
291
292
 
292
293
  try:
293
294
  j = tuple(j)
294
- except Exception as e:
295
+ except Exception:
295
296
  return False, response.text
296
297
 
297
298
  if debug:
@@ -313,12 +314,12 @@ def sync_pipe(
313
314
 
314
315
  def delete_pipe(
315
316
  self,
316
- pipe: Optional[meerschaum.Pipe] = None,
317
+ pipe: Optional[mrsm.Pipe] = None,
317
318
  debug: bool = None,
318
319
  ) -> SuccessTuple:
319
320
  """Delete a Pipe and drop its table."""
320
321
  if pipe is None:
321
- error(f"Pipe cannot be None.")
322
+ error("Pipe cannot be None.")
322
323
  r_url = pipe_r_url(pipe)
323
324
  response = self.delete(
324
325
  r_url + '/delete',
@@ -339,7 +340,7 @@ def delete_pipe(
339
340
 
340
341
  def get_pipe_data(
341
342
  self,
342
- pipe: meerschaum.Pipe,
343
+ pipe: mrsm.Pipe,
343
344
  select_columns: Optional[List[str]] = None,
344
345
  omit_columns: Optional[List[str]] = None,
345
346
  begin: Union[str, datetime, int, None] = None,
@@ -351,7 +352,6 @@ def get_pipe_data(
351
352
  ) -> Union[pandas.DataFrame, None]:
352
353
  """Fetch data from the API."""
353
354
  r_url = pipe_r_url(pipe)
354
- chunks_list = []
355
355
  while True:
356
356
  try:
357
357
  response = self.get(
@@ -375,12 +375,19 @@ def get_pipe_data(
375
375
  return False, j['detail']
376
376
  break
377
377
 
378
- from meerschaum.utils.packages import import_pandas
379
378
  from meerschaum.utils.dataframe import parse_df_datetimes, add_missing_cols_to_df
380
379
  from meerschaum.utils.dtypes import are_dtypes_equal
381
- pd = import_pandas()
382
380
  try:
383
- df = pd.read_json(StringIO(response.text))
381
+ df = parse_df_datetimes(
382
+ j,
383
+ ignore_cols=[
384
+ col
385
+ for col, dtype in pipe.dtypes.items()
386
+ if not are_dtypes_equal(str(dtype), 'datetime')
387
+ ],
388
+ strip_timezone=(pipe.tzinfo is None),
389
+ debug=debug,
390
+ )
384
391
  except Exception as e:
385
392
  warn(f"Failed to parse response for {pipe}:\n{e}")
386
393
  return None
@@ -388,16 +395,6 @@ def get_pipe_data(
388
395
  if len(df.columns) == 0:
389
396
  return add_missing_cols_to_df(df, pipe.dtypes)
390
397
 
391
- df = parse_df_datetimes(
392
- df,
393
- ignore_cols = [
394
- col
395
- for col, dtype in pipe.dtypes.items()
396
- if not are_dtypes_equal(str(dtype), 'datetime')
397
- ],
398
- strip_timezone=(pipe.tzinfo is None),
399
- debug=debug,
400
- )
401
398
  return df
402
399
 
403
400
 
@@ -107,7 +107,7 @@ flavor_configs = {
107
107
  },
108
108
  },
109
109
  'oracle': {
110
- 'engine': 'oracle+cx_oracle',
110
+ 'engine': 'oracle+oracledb',
111
111
  'create_engine': default_create_engine_args,
112
112
  'omit_create_engine': {'method',},
113
113
  'to_sql': {
@@ -164,7 +164,7 @@ install_flavor_drivers = {
164
164
  'citus': ['psycopg'],
165
165
  'cockroachdb': ['psycopg', 'sqlalchemy_cockroachdb', 'sqlalchemy_cockroachdb.psycopg'],
166
166
  'mssql': ['pyodbc'],
167
- 'oracle': ['cx_Oracle'],
167
+ 'oracle': ['oracledb'],
168
168
  }
169
169
  require_patching_flavors = {'cockroachdb': [('sqlalchemy-cockroachdb', 'sqlalchemy_cockroachdb')]}
170
170
 
@@ -239,7 +239,7 @@ def create_engine(
239
239
  self._sys_config['create_engine'] = {}
240
240
  if 'connect_args' not in self._sys_config['create_engine']:
241
241
  self._sys_config['create_engine']['connect_args'] = {}
242
- self._sys_config['create_engine']['connect_args'].update({"check_same_thread" : False})
242
+ self._sys_config['create_engine']['connect_args'].update({"check_same_thread": False})
243
243
  else:
244
244
  engine_str = (
245
245
  _engine + "://" + (_username if _username is not None else '') +
@@ -9,8 +9,7 @@ Define utilities for instance connectors.
9
9
  import time
10
10
  from datetime import datetime, timezone, timedelta
11
11
  import meerschaum as mrsm
12
- from meerschaum.utils.typing import Dict, SuccessTuple, Optional, Union, List
13
- from meerschaum.utils.warnings import warn
12
+ from meerschaum.utils.typing import Dict, SuccessTuple, Union, List
14
13
 
15
14
 
16
15
  _in_memory_temp_tables: Dict[str, bool] = {}
@@ -28,9 +27,9 @@ def _log_temporary_tables_creation(
28
27
  from meerschaum.connectors.sql.tables import get_tables
29
28
  sqlalchemy = mrsm.attempt_import('sqlalchemy')
30
29
  temp_tables_table = get_tables(
31
- mrsm_instance = self,
32
- create = create,
33
- debug = debug,
30
+ mrsm_instance=self,
31
+ create=create,
32
+ debug=debug,
34
33
  )['temp_tables']
35
34
  if isinstance(tables, str):
36
35
  tables = [tables]
@@ -72,7 +71,9 @@ def _drop_temporary_table(
72
71
  return True, "Success"
73
72
 
74
73
  drop_query = f"DROP TABLE {if_exists} " + sql_item_name(
75
- table, self.flavor, schema=self.internal_schema
74
+ table,
75
+ self.flavor,
76
+ schema=self.internal_schema
76
77
  )
77
78
  drop_success = self.exec(drop_query, silent=True, debug=debug) is not None
78
79
  drop_msg = "Success" if drop_success else f"Failed to drop temporary table '{table}'."
@@ -83,7 +84,6 @@ def _drop_temporary_tables(self, debug: bool = False) -> SuccessTuple:
83
84
  """
84
85
  Drop all tables in the internal schema that are marked as ready to be dropped.
85
86
  """
86
- from meerschaum.utils.sql import sql_item_name, table_exists
87
87
  from meerschaum.utils.misc import items_str
88
88
  from meerschaum.connectors.sql.tables import get_tables
89
89
  sqlalchemy = mrsm.attempt_import('sqlalchemy')
@@ -141,16 +141,15 @@ def _drop_temporary_tables(self, debug: bool = False) -> SuccessTuple:
141
141
 
142
142
 
143
143
  def _drop_old_temporary_tables(
144
- self,
145
- refresh: bool = True,
146
- debug: bool = False,
147
- ) -> SuccessTuple:
144
+ self,
145
+ refresh: bool = True,
146
+ debug: bool = False,
147
+ ) -> SuccessTuple:
148
148
  """
149
149
  Drop temporary tables older than the configured interval (24 hours by default).
150
150
  """
151
151
  from meerschaum.config import get_config
152
152
  from meerschaum.connectors.sql.tables import get_tables
153
- from meerschaum.utils.misc import items_str
154
153
  sqlalchemy = mrsm.attempt_import('sqlalchemy')
155
154
  temp_tables_table = get_tables(mrsm_instance=self, create=False, debug=debug)['temp_tables']
156
155
  last_check = getattr(self, '_stale_temporary_tables_check_timestamp', 0)