meerschaum 2.2.5.dev0__py3-none-any.whl → 2.2.5.dev2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/_internal/arguments/_parse_arguments.py +23 -14
- meerschaum/_internal/arguments/_parser.py +4 -1
- meerschaum/_internal/shell/Shell.py +0 -3
- meerschaum/actions/bootstrap.py +14 -235
- meerschaum/actions/edit.py +98 -15
- meerschaum/actions/uninstall.py +24 -29
- meerschaum/config/__init__.py +16 -6
- meerschaum/config/_version.py +1 -1
- meerschaum/core/Pipe/_fetch.py +25 -21
- meerschaum/core/Pipe/_sync.py +89 -59
- meerschaum/plugins/bootstrap.py +333 -0
- meerschaum/utils/formatting/__init__.py +22 -10
- meerschaum/utils/prompt.py +11 -4
- {meerschaum-2.2.5.dev0.dist-info → meerschaum-2.2.5.dev2.dist-info}/METADATA +1 -1
- {meerschaum-2.2.5.dev0.dist-info → meerschaum-2.2.5.dev2.dist-info}/RECORD +21 -20
- {meerschaum-2.2.5.dev0.dist-info → meerschaum-2.2.5.dev2.dist-info}/LICENSE +0 -0
- {meerschaum-2.2.5.dev0.dist-info → meerschaum-2.2.5.dev2.dist-info}/NOTICE +0 -0
- {meerschaum-2.2.5.dev0.dist-info → meerschaum-2.2.5.dev2.dist-info}/WHEEL +0 -0
- {meerschaum-2.2.5.dev0.dist-info → meerschaum-2.2.5.dev2.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.2.5.dev0.dist-info → meerschaum-2.2.5.dev2.dist-info}/top_level.txt +0 -0
- {meerschaum-2.2.5.dev0.dist-info → meerschaum-2.2.5.dev2.dist-info}/zip-safe +0 -0
meerschaum/config/__init__.py
CHANGED
@@ -12,7 +12,7 @@ from __future__ import annotations
|
|
12
12
|
import os, shutil, sys, pathlib, copy
|
13
13
|
from meerschaum.utils.typing import Any, Dict, Optional, Union
|
14
14
|
from meerschaum.utils.threading import RLock
|
15
|
-
from meerschaum.utils.warnings import warn
|
15
|
+
from meerschaum.utils.warnings import warn, error
|
16
16
|
|
17
17
|
from meerschaum.config._version import __version__
|
18
18
|
from meerschaum.config._edit import edit_config, write_config
|
@@ -239,17 +239,28 @@ def get_config(
|
|
239
239
|
return c
|
240
240
|
|
241
241
|
|
242
|
-
def get_plugin_config(
|
242
|
+
def get_plugin_config(
|
243
|
+
*keys: str,
|
244
|
+
warn: bool = False,
|
245
|
+
**kw: Any
|
246
|
+
) -> Optional[Any]:
|
243
247
|
"""
|
244
248
|
This may only be called from within a Meerschaum plugin.
|
245
249
|
See `meerschaum.config.get_config` for arguments.
|
246
250
|
"""
|
247
|
-
from meerschaum.utils.warnings import warn, error
|
248
251
|
from meerschaum.plugins import _get_parent_plugin
|
249
252
|
parent_plugin_name = _get_parent_plugin(2)
|
250
253
|
if parent_plugin_name is None:
|
251
|
-
error(
|
252
|
-
|
254
|
+
error(
|
255
|
+
"You may only call `get_plugin_config()` "
|
256
|
+
"from within a Meerschaum plugin."
|
257
|
+
)
|
258
|
+
|
259
|
+
return get_config(
|
260
|
+
*(['plugins', parent_plugin_name] + list(keys)),
|
261
|
+
warn=warn,
|
262
|
+
**kw
|
263
|
+
)
|
253
264
|
|
254
265
|
|
255
266
|
def write_plugin_config(
|
@@ -259,7 +270,6 @@ def write_plugin_config(
|
|
259
270
|
"""
|
260
271
|
Write a plugin's configuration dictionary.
|
261
272
|
"""
|
262
|
-
from meerschaum.utils.warnings import warn, error
|
263
273
|
from meerschaum.plugins import _get_parent_plugin
|
264
274
|
parent_plugin_name = _get_parent_plugin(2)
|
265
275
|
if parent_plugin_name is None:
|
meerschaum/config/_version.py
CHANGED
meerschaum/core/Pipe/_fetch.py
CHANGED
@@ -12,6 +12,7 @@ import meerschaum as mrsm
|
|
12
12
|
from meerschaum.utils.typing import Optional, Any, Union, SuccessTuple, Iterator
|
13
13
|
from meerschaum.config import get_config
|
14
14
|
from meerschaum.utils.warnings import warn
|
15
|
+
from meerschaum.utils.misc import filter_keywords
|
15
16
|
|
16
17
|
def fetch(
|
17
18
|
self,
|
@@ -75,25 +76,28 @@ def fetch(
|
|
75
76
|
with mrsm.Venv(get_connector_plugin(self.connector)):
|
76
77
|
df = self.connector.fetch(
|
77
78
|
self,
|
78
|
-
|
79
|
-
self,
|
80
|
-
begin
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
79
|
+
**filter_keywords(
|
80
|
+
self.connector.fetch,
|
81
|
+
begin=_determine_begin(
|
82
|
+
self,
|
83
|
+
begin,
|
84
|
+
check_existing=check_existing,
|
85
|
+
debug=debug,
|
86
|
+
),
|
87
|
+
end=end,
|
88
|
+
chunk_hook=_chunk_hook,
|
89
|
+
debug=debug,
|
90
|
+
**kw
|
91
|
+
)
|
88
92
|
)
|
89
93
|
return df
|
90
94
|
|
91
95
|
|
92
96
|
def get_backtrack_interval(
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
+
self,
|
98
|
+
check_existing: bool = True,
|
99
|
+
debug: bool = False,
|
100
|
+
) -> Union[timedelta, int]:
|
97
101
|
"""
|
98
102
|
Get the chunk interval to use for this pipe.
|
99
103
|
|
@@ -127,17 +131,17 @@ def get_backtrack_interval(
|
|
127
131
|
|
128
132
|
|
129
133
|
def _determine_begin(
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
134
|
+
pipe: mrsm.Pipe,
|
135
|
+
begin: Union[datetime, int, str, None] = '',
|
136
|
+
check_existing: bool = True,
|
137
|
+
debug: bool = False,
|
138
|
+
) -> Union[datetime, int, None]:
|
135
139
|
"""
|
136
140
|
Apply the backtrack interval if `--begin` is not provided.
|
137
141
|
|
138
142
|
Parameters
|
139
143
|
----------
|
140
|
-
begin: Union[datetime, int, str], default ''
|
144
|
+
begin: Union[datetime, int, str, None], default ''
|
141
145
|
The provided begin timestamp.
|
142
146
|
|
143
147
|
check_existing: bool, default True
|
@@ -160,4 +164,4 @@ def _determine_begin(
|
|
160
164
|
return sync_time - backtrack_interval
|
161
165
|
except Exception as e:
|
162
166
|
warn(f"Unable to substract backtrack interval {backtrack_interval} from {sync_time}.")
|
163
|
-
|
167
|
+
return sync_time
|
meerschaum/core/Pipe/_sync.py
CHANGED
@@ -34,29 +34,29 @@ class InferFetch:
|
|
34
34
|
MRSM_INFER_FETCH: bool = True
|
35
35
|
|
36
36
|
def sync(
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
37
|
+
self,
|
38
|
+
df: Union[
|
39
|
+
pd.DataFrame,
|
40
|
+
Dict[str, List[Any]],
|
41
|
+
List[Dict[str, Any]],
|
42
|
+
InferFetch
|
43
|
+
] = InferFetch,
|
44
|
+
begin: Union[datetime, int, str, None] = '',
|
45
|
+
end: Union[datetime, int, None] = None,
|
46
|
+
force: bool = False,
|
47
|
+
retries: int = 10,
|
48
|
+
min_seconds: int = 1,
|
49
|
+
check_existing: bool = True,
|
50
|
+
blocking: bool = True,
|
51
|
+
workers: Optional[int] = None,
|
52
|
+
callback: Optional[Callable[[Tuple[bool, str]], Any]] = None,
|
53
|
+
error_callback: Optional[Callable[[Exception], Any]] = None,
|
54
|
+
chunksize: Optional[int] = -1,
|
55
|
+
sync_chunks: bool = True,
|
56
|
+
debug: bool = False,
|
57
|
+
_inplace: bool = True,
|
58
|
+
**kw: Any
|
59
|
+
) -> SuccessTuple:
|
60
60
|
"""
|
61
61
|
Fetch new data from the source and update the pipe's table with new data.
|
62
62
|
|
@@ -125,7 +125,7 @@ def sync(
|
|
125
125
|
from meerschaum.utils.formatting import get_console
|
126
126
|
from meerschaum.utils.venv import Venv
|
127
127
|
from meerschaum.connectors import get_connector_plugin
|
128
|
-
from meerschaum.utils.misc import df_is_chunk_generator
|
128
|
+
from meerschaum.utils.misc import df_is_chunk_generator, filter_keywords
|
129
129
|
from meerschaum.utils.pool import get_pool
|
130
130
|
from meerschaum.config import get_config
|
131
131
|
|
@@ -186,7 +186,7 @@ def sync(
|
|
186
186
|
### use that instead.
|
187
187
|
### NOTE: The DataFrame must be omitted for the plugin sync method to apply.
|
188
188
|
### If a DataFrame is provided, continue as expected.
|
189
|
-
if hasattr(df, 'MRSM_INFER_FETCH'):
|
189
|
+
if hasattr(df, 'MRSM_INFER_FETCH'):
|
190
190
|
try:
|
191
191
|
if p.connector is None:
|
192
192
|
msg = f"{p} does not have a valid connector."
|
@@ -194,7 +194,7 @@ def sync(
|
|
194
194
|
msg += f"\n Perhaps {p.connector_keys} has a syntax error?"
|
195
195
|
p._exists = None
|
196
196
|
return False, msg
|
197
|
-
except Exception
|
197
|
+
except Exception:
|
198
198
|
p._exists = None
|
199
199
|
return False, f"Unable to create the connector for {p}."
|
200
200
|
|
@@ -210,14 +210,28 @@ def sync(
|
|
210
210
|
):
|
211
211
|
with Venv(get_connector_plugin(self.instance_connector)):
|
212
212
|
p._exists = None
|
213
|
-
return self.instance_connector.sync_pipe_inplace(
|
213
|
+
return self.instance_connector.sync_pipe_inplace(
|
214
|
+
p,
|
215
|
+
**filter_keywords(
|
216
|
+
p.instance_connector.sync_pipe_inplace,
|
217
|
+
debug=debug,
|
218
|
+
**kw
|
219
|
+
)
|
220
|
+
)
|
214
221
|
|
215
222
|
|
216
223
|
### Activate and invoke `sync(pipe)` for plugin connectors with `sync` methods.
|
217
224
|
try:
|
218
225
|
if getattr(p.connector, 'sync', None) is not None:
|
219
226
|
with Venv(get_connector_plugin(p.connector), debug=debug):
|
220
|
-
return_tuple = p.connector.sync(
|
227
|
+
return_tuple = p.connector.sync(
|
228
|
+
p,
|
229
|
+
**filter_keywords(
|
230
|
+
p.connector.sync,
|
231
|
+
debug=debug,
|
232
|
+
**kw
|
233
|
+
)
|
234
|
+
)
|
221
235
|
p._exists = None
|
222
236
|
if not isinstance(return_tuple, tuple):
|
223
237
|
return_tuple = (
|
@@ -237,13 +251,19 @@ def sync(
|
|
237
251
|
### Fetch the dataframe from the connector's `fetch()` method.
|
238
252
|
try:
|
239
253
|
with Venv(get_connector_plugin(p.connector), debug=debug):
|
240
|
-
df = p.fetch(
|
254
|
+
df = p.fetch(
|
255
|
+
**filter_keywords(
|
256
|
+
p.fetch,
|
257
|
+
debug=debug,
|
258
|
+
**kw
|
259
|
+
)
|
260
|
+
)
|
241
261
|
|
242
262
|
except Exception as e:
|
243
263
|
get_console().print_exception(
|
244
|
-
suppress
|
245
|
-
'meerschaum/core/Pipe/_sync.py',
|
246
|
-
'meerschaum/core/Pipe/_fetch.py',
|
264
|
+
suppress=[
|
265
|
+
'meerschaum/core/Pipe/_sync.py',
|
266
|
+
'meerschaum/core/Pipe/_fetch.py',
|
247
267
|
]
|
248
268
|
)
|
249
269
|
msg = f"Failed to fetch data from {p.connector}:\n {e}"
|
@@ -289,7 +309,7 @@ def sync(
|
|
289
309
|
if not chunk_success:
|
290
310
|
return chunk_success, f"Unable to sync initial chunk for {p}:\n{chunk_msg}"
|
291
311
|
if debug:
|
292
|
-
dprint(
|
312
|
+
dprint("Successfully synced the first chunk, attemping the rest...")
|
293
313
|
|
294
314
|
failed_chunks = []
|
295
315
|
def _process_chunk(_chunk):
|
@@ -309,7 +329,6 @@ def sync(
|
|
309
329
|
)
|
310
330
|
)
|
311
331
|
|
312
|
-
|
313
332
|
results = sorted(
|
314
333
|
[(chunk_success, chunk_msg)] + (
|
315
334
|
list(pool.imap(_process_chunk, df))
|
@@ -329,7 +348,7 @@ def sync(
|
|
329
348
|
retry_success = True
|
330
349
|
if not success and any(success_bools):
|
331
350
|
if debug:
|
332
|
-
dprint(
|
351
|
+
dprint("Retrying failed chunks...")
|
333
352
|
chunks_to_retry = [c for c in failed_chunks]
|
334
353
|
failed_chunks = []
|
335
354
|
for chunk in chunks_to_retry:
|
@@ -361,9 +380,9 @@ def sync(
|
|
361
380
|
while run:
|
362
381
|
with Venv(get_connector_plugin(self.instance_connector)):
|
363
382
|
return_tuple = p.instance_connector.sync_pipe(
|
364
|
-
pipe
|
365
|
-
df
|
366
|
-
debug
|
383
|
+
pipe=p,
|
384
|
+
df=df,
|
385
|
+
debug=debug,
|
367
386
|
**kw
|
368
387
|
)
|
369
388
|
_retries += 1
|
@@ -382,7 +401,7 @@ def sync(
|
|
382
401
|
_checkpoint(**kw)
|
383
402
|
if self.cache_pipe is not None:
|
384
403
|
if debug:
|
385
|
-
dprint(
|
404
|
+
dprint("Caching retrieved dataframe.", **kw)
|
386
405
|
_sync_cache_tuple = self.cache_pipe.sync(df, debug=debug, **kw)
|
387
406
|
if not _sync_cache_tuple[0]:
|
388
407
|
warn(f"Failed to sync local cache for {self}.")
|
@@ -395,10 +414,10 @@ def sync(
|
|
395
414
|
return _sync(self, df = df)
|
396
415
|
|
397
416
|
from meerschaum.utils.threading import Thread
|
398
|
-
def default_callback(result_tuple
|
417
|
+
def default_callback(result_tuple: SuccessTuple):
|
399
418
|
dprint(f"Asynchronous result from {self}: {result_tuple}", **kw)
|
400
419
|
|
401
|
-
def default_error_callback(x
|
420
|
+
def default_error_callback(x: Exception):
|
402
421
|
dprint(f"Error received for {self}: {x}", **kw)
|
403
422
|
|
404
423
|
if callback is None and debug:
|
@@ -407,12 +426,12 @@ def sync(
|
|
407
426
|
error_callback = default_error_callback
|
408
427
|
try:
|
409
428
|
thread = Thread(
|
410
|
-
target
|
411
|
-
args
|
412
|
-
kwargs
|
413
|
-
daemon
|
414
|
-
callback
|
415
|
-
error_callback
|
429
|
+
target=_sync,
|
430
|
+
args=(self,),
|
431
|
+
kwargs={'df': df},
|
432
|
+
daemon=False,
|
433
|
+
callback=callback,
|
434
|
+
error_callback=error_callback,
|
416
435
|
)
|
417
436
|
thread.start()
|
418
437
|
except Exception as e:
|
@@ -424,12 +443,13 @@ def sync(
|
|
424
443
|
|
425
444
|
|
426
445
|
def get_sync_time(
|
427
|
-
|
428
|
-
|
429
|
-
|
430
|
-
|
431
|
-
|
432
|
-
|
446
|
+
self,
|
447
|
+
params: Optional[Dict[str, Any]] = None,
|
448
|
+
newest: bool = True,
|
449
|
+
apply_backtrack_interval: bool = False,
|
450
|
+
round_down: bool = False,
|
451
|
+
debug: bool = False
|
452
|
+
) -> Union['datetime', None]:
|
433
453
|
"""
|
434
454
|
Get the most recent datetime value for a Pipe.
|
435
455
|
|
@@ -443,6 +463,9 @@ def get_sync_time(
|
|
443
463
|
If `True`, get the most recent datetime (honoring `params`).
|
444
464
|
If `False`, get the oldest datetime (`ASC` instead of `DESC`).
|
445
465
|
|
466
|
+
apply_backtrack_interval: bool, default False
|
467
|
+
If `True`, subtract the backtrack interval from the sync time.
|
468
|
+
|
446
469
|
round_down: bool, default False
|
447
470
|
If `True`, round down the datetime value to the nearest minute.
|
448
471
|
|
@@ -461,15 +484,22 @@ def get_sync_time(
|
|
461
484
|
with Venv(get_connector_plugin(self.instance_connector)):
|
462
485
|
sync_time = self.instance_connector.get_sync_time(
|
463
486
|
self,
|
464
|
-
params
|
465
|
-
newest
|
466
|
-
debug
|
487
|
+
params=params,
|
488
|
+
newest=newest,
|
489
|
+
debug=debug,
|
467
490
|
)
|
468
491
|
|
469
|
-
if
|
470
|
-
|
492
|
+
if round_down and isinstance(sync_time, datetime):
|
493
|
+
sync_time = round_time(sync_time, timedelta(minutes=1))
|
494
|
+
|
495
|
+
if apply_backtrack_interval and sync_time is not None:
|
496
|
+
backtrack_interval = self.get_backtrack_interval(debug=debug)
|
497
|
+
try:
|
498
|
+
sync_time -= backtrack_interval
|
499
|
+
except Exception as e:
|
500
|
+
warn(f"Failed to apply backtrack interval:\n{e}")
|
471
501
|
|
472
|
-
return
|
502
|
+
return sync_time
|
473
503
|
|
474
504
|
|
475
505
|
def exists(
|