meerschaum 2.2.3__py3-none-any.whl → 2.2.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/__init__.py +4 -1
- meerschaum/_internal/arguments/_parse_arguments.py +23 -14
- meerschaum/_internal/arguments/_parser.py +4 -2
- meerschaum/_internal/docs/index.py +513 -110
- meerschaum/_internal/entry.py +2 -4
- meerschaum/_internal/shell/Shell.py +0 -3
- meerschaum/actions/__init__.py +5 -1
- meerschaum/actions/bootstrap.py +32 -7
- meerschaum/actions/delete.py +62 -0
- meerschaum/actions/edit.py +98 -15
- meerschaum/actions/python.py +45 -14
- meerschaum/actions/show.py +39 -4
- meerschaum/actions/stack.py +12 -12
- meerschaum/actions/uninstall.py +24 -29
- meerschaum/api/__init__.py +0 -1
- meerschaum/api/_oauth2.py +17 -0
- meerschaum/api/dash/__init__.py +0 -1
- meerschaum/api/dash/callbacks/custom.py +1 -1
- meerschaum/api/dash/plugins.py +5 -6
- meerschaum/api/routes/_login.py +23 -7
- meerschaum/config/__init__.py +16 -6
- meerschaum/config/_edit.py +1 -1
- meerschaum/config/_paths.py +3 -0
- meerschaum/config/_version.py +1 -1
- meerschaum/config/stack/__init__.py +3 -1
- meerschaum/connectors/Connector.py +7 -2
- meerschaum/connectors/__init__.py +7 -5
- meerschaum/core/Pipe/_data.py +23 -15
- meerschaum/core/Pipe/_deduplicate.py +1 -1
- meerschaum/core/Pipe/_dtypes.py +5 -0
- meerschaum/core/Pipe/_fetch.py +26 -20
- meerschaum/core/Pipe/_sync.py +96 -61
- meerschaum/plugins/__init__.py +1 -1
- meerschaum/plugins/bootstrap.py +333 -0
- meerschaum/utils/daemon/Daemon.py +14 -3
- meerschaum/utils/daemon/FileDescriptorInterceptor.py +21 -14
- meerschaum/utils/daemon/RotatingFile.py +21 -18
- meerschaum/utils/dataframe.py +12 -4
- meerschaum/utils/debug.py +9 -15
- meerschaum/utils/formatting/__init__.py +23 -10
- meerschaum/utils/misc.py +117 -11
- meerschaum/utils/packages/_packages.py +1 -0
- meerschaum/utils/prompt.py +64 -21
- meerschaum/utils/typing.py +1 -0
- meerschaum/utils/warnings.py +9 -1
- meerschaum/utils/yaml.py +32 -1
- {meerschaum-2.2.3.dist-info → meerschaum-2.2.5.dist-info}/METADATA +5 -1
- {meerschaum-2.2.3.dist-info → meerschaum-2.2.5.dist-info}/RECORD +54 -53
- {meerschaum-2.2.3.dist-info → meerschaum-2.2.5.dist-info}/WHEEL +1 -1
- {meerschaum-2.2.3.dist-info → meerschaum-2.2.5.dist-info}/LICENSE +0 -0
- {meerschaum-2.2.3.dist-info → meerschaum-2.2.5.dist-info}/NOTICE +0 -0
- {meerschaum-2.2.3.dist-info → meerschaum-2.2.5.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.2.3.dist-info → meerschaum-2.2.5.dist-info}/top_level.txt +0 -0
- {meerschaum-2.2.3.dist-info → meerschaum-2.2.5.dist-info}/zip-safe +0 -0
meerschaum/core/Pipe/_sync.py
CHANGED
@@ -14,7 +14,9 @@ import threading
|
|
14
14
|
import multiprocessing
|
15
15
|
import functools
|
16
16
|
from datetime import datetime, timedelta
|
17
|
+
from typing import TYPE_CHECKING
|
17
18
|
|
19
|
+
import meerschaum as mrsm
|
18
20
|
from meerschaum.utils.typing import (
|
19
21
|
Union,
|
20
22
|
Optional,
|
@@ -26,37 +28,40 @@ from meerschaum.utils.typing import (
|
|
26
28
|
List,
|
27
29
|
Iterable,
|
28
30
|
Generator,
|
29
|
-
Iterator,
|
30
31
|
)
|
31
32
|
from meerschaum.utils.warnings import warn, error
|
32
33
|
|
34
|
+
if TYPE_CHECKING:
|
35
|
+
pd = mrsm.attempt_import('pandas')
|
36
|
+
|
33
37
|
class InferFetch:
|
34
38
|
MRSM_INFER_FETCH: bool = True
|
35
39
|
|
40
|
+
|
36
41
|
def sync(
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
42
|
+
self,
|
43
|
+
df: Union[
|
44
|
+
pd.DataFrame,
|
45
|
+
Dict[str, List[Any]],
|
46
|
+
List[Dict[str, Any]],
|
47
|
+
InferFetch
|
48
|
+
] = InferFetch,
|
49
|
+
begin: Union[datetime, int, str, None] = '',
|
50
|
+
end: Union[datetime, int, None] = None,
|
51
|
+
force: bool = False,
|
52
|
+
retries: int = 10,
|
53
|
+
min_seconds: int = 1,
|
54
|
+
check_existing: bool = True,
|
55
|
+
blocking: bool = True,
|
56
|
+
workers: Optional[int] = None,
|
57
|
+
callback: Optional[Callable[[Tuple[bool, str]], Any]] = None,
|
58
|
+
error_callback: Optional[Callable[[Exception], Any]] = None,
|
59
|
+
chunksize: Optional[int] = -1,
|
60
|
+
sync_chunks: bool = True,
|
61
|
+
debug: bool = False,
|
62
|
+
_inplace: bool = True,
|
63
|
+
**kw: Any
|
64
|
+
) -> SuccessTuple:
|
60
65
|
"""
|
61
66
|
Fetch new data from the source and update the pipe's table with new data.
|
62
67
|
|
@@ -125,7 +130,7 @@ def sync(
|
|
125
130
|
from meerschaum.utils.formatting import get_console
|
126
131
|
from meerschaum.utils.venv import Venv
|
127
132
|
from meerschaum.connectors import get_connector_plugin
|
128
|
-
from meerschaum.utils.misc import df_is_chunk_generator
|
133
|
+
from meerschaum.utils.misc import df_is_chunk_generator, filter_keywords, filter_arguments
|
129
134
|
from meerschaum.utils.pool import get_pool
|
130
135
|
from meerschaum.config import get_config
|
131
136
|
|
@@ -186,7 +191,7 @@ def sync(
|
|
186
191
|
### use that instead.
|
187
192
|
### NOTE: The DataFrame must be omitted for the plugin sync method to apply.
|
188
193
|
### If a DataFrame is provided, continue as expected.
|
189
|
-
if hasattr(df, 'MRSM_INFER_FETCH'):
|
194
|
+
if hasattr(df, 'MRSM_INFER_FETCH'):
|
190
195
|
try:
|
191
196
|
if p.connector is None:
|
192
197
|
msg = f"{p} does not have a valid connector."
|
@@ -194,7 +199,7 @@ def sync(
|
|
194
199
|
msg += f"\n Perhaps {p.connector_keys} has a syntax error?"
|
195
200
|
p._exists = None
|
196
201
|
return False, msg
|
197
|
-
except Exception
|
202
|
+
except Exception:
|
198
203
|
p._exists = None
|
199
204
|
return False, f"Unable to create the connector for {p}."
|
200
205
|
|
@@ -210,14 +215,28 @@ def sync(
|
|
210
215
|
):
|
211
216
|
with Venv(get_connector_plugin(self.instance_connector)):
|
212
217
|
p._exists = None
|
213
|
-
|
214
|
-
|
218
|
+
_args, _kwargs = filter_arguments(
|
219
|
+
p.instance_connector.sync_pipe_inplace,
|
220
|
+
p,
|
221
|
+
debug=debug,
|
222
|
+
**kw
|
223
|
+
)
|
224
|
+
return self.instance_connector.sync_pipe_inplace(
|
225
|
+
*_args,
|
226
|
+
**_kwargs
|
227
|
+
)
|
215
228
|
|
216
229
|
### Activate and invoke `sync(pipe)` for plugin connectors with `sync` methods.
|
217
230
|
try:
|
218
231
|
if getattr(p.connector, 'sync', None) is not None:
|
219
232
|
with Venv(get_connector_plugin(p.connector), debug=debug):
|
220
|
-
|
233
|
+
_args, _kwargs = filter_arguments(
|
234
|
+
p.connector.sync,
|
235
|
+
p,
|
236
|
+
debug=debug,
|
237
|
+
**kw
|
238
|
+
)
|
239
|
+
return_tuple = p.connector.sync(*_args, **_kwargs)
|
221
240
|
p._exists = None
|
222
241
|
if not isinstance(return_tuple, tuple):
|
223
242
|
return_tuple = (
|
@@ -237,13 +256,19 @@ def sync(
|
|
237
256
|
### Fetch the dataframe from the connector's `fetch()` method.
|
238
257
|
try:
|
239
258
|
with Venv(get_connector_plugin(p.connector), debug=debug):
|
240
|
-
df = p.fetch(
|
259
|
+
df = p.fetch(
|
260
|
+
**filter_keywords(
|
261
|
+
p.fetch,
|
262
|
+
debug=debug,
|
263
|
+
**kw
|
264
|
+
)
|
265
|
+
)
|
241
266
|
|
242
267
|
except Exception as e:
|
243
268
|
get_console().print_exception(
|
244
|
-
suppress
|
245
|
-
'meerschaum/core/Pipe/_sync.py',
|
246
|
-
'meerschaum/core/Pipe/_fetch.py',
|
269
|
+
suppress=[
|
270
|
+
'meerschaum/core/Pipe/_sync.py',
|
271
|
+
'meerschaum/core/Pipe/_fetch.py',
|
247
272
|
]
|
248
273
|
)
|
249
274
|
msg = f"Failed to fetch data from {p.connector}:\n {e}"
|
@@ -289,7 +314,7 @@ def sync(
|
|
289
314
|
if not chunk_success:
|
290
315
|
return chunk_success, f"Unable to sync initial chunk for {p}:\n{chunk_msg}"
|
291
316
|
if debug:
|
292
|
-
dprint(
|
317
|
+
dprint("Successfully synced the first chunk, attemping the rest...")
|
293
318
|
|
294
319
|
failed_chunks = []
|
295
320
|
def _process_chunk(_chunk):
|
@@ -309,7 +334,6 @@ def sync(
|
|
309
334
|
)
|
310
335
|
)
|
311
336
|
|
312
|
-
|
313
337
|
results = sorted(
|
314
338
|
[(chunk_success, chunk_msg)] + (
|
315
339
|
list(pool.imap(_process_chunk, df))
|
@@ -329,7 +353,7 @@ def sync(
|
|
329
353
|
retry_success = True
|
330
354
|
if not success and any(success_bools):
|
331
355
|
if debug:
|
332
|
-
dprint(
|
356
|
+
dprint("Retrying failed chunks...")
|
333
357
|
chunks_to_retry = [c for c in failed_chunks]
|
334
358
|
failed_chunks = []
|
335
359
|
for chunk in chunks_to_retry:
|
@@ -361,9 +385,9 @@ def sync(
|
|
361
385
|
while run:
|
362
386
|
with Venv(get_connector_plugin(self.instance_connector)):
|
363
387
|
return_tuple = p.instance_connector.sync_pipe(
|
364
|
-
pipe
|
365
|
-
df
|
366
|
-
debug
|
388
|
+
pipe=p,
|
389
|
+
df=df,
|
390
|
+
debug=debug,
|
367
391
|
**kw
|
368
392
|
)
|
369
393
|
_retries += 1
|
@@ -382,7 +406,7 @@ def sync(
|
|
382
406
|
_checkpoint(**kw)
|
383
407
|
if self.cache_pipe is not None:
|
384
408
|
if debug:
|
385
|
-
dprint(
|
409
|
+
dprint("Caching retrieved dataframe.", **kw)
|
386
410
|
_sync_cache_tuple = self.cache_pipe.sync(df, debug=debug, **kw)
|
387
411
|
if not _sync_cache_tuple[0]:
|
388
412
|
warn(f"Failed to sync local cache for {self}.")
|
@@ -395,10 +419,10 @@ def sync(
|
|
395
419
|
return _sync(self, df = df)
|
396
420
|
|
397
421
|
from meerschaum.utils.threading import Thread
|
398
|
-
def default_callback(result_tuple
|
422
|
+
def default_callback(result_tuple: SuccessTuple):
|
399
423
|
dprint(f"Asynchronous result from {self}: {result_tuple}", **kw)
|
400
424
|
|
401
|
-
def default_error_callback(x
|
425
|
+
def default_error_callback(x: Exception):
|
402
426
|
dprint(f"Error received for {self}: {x}", **kw)
|
403
427
|
|
404
428
|
if callback is None and debug:
|
@@ -407,12 +431,12 @@ def sync(
|
|
407
431
|
error_callback = default_error_callback
|
408
432
|
try:
|
409
433
|
thread = Thread(
|
410
|
-
target
|
411
|
-
args
|
412
|
-
kwargs
|
413
|
-
daemon
|
414
|
-
callback
|
415
|
-
error_callback
|
434
|
+
target=_sync,
|
435
|
+
args=(self,),
|
436
|
+
kwargs={'df': df},
|
437
|
+
daemon=False,
|
438
|
+
callback=callback,
|
439
|
+
error_callback=error_callback,
|
416
440
|
)
|
417
441
|
thread.start()
|
418
442
|
except Exception as e:
|
@@ -424,12 +448,13 @@ def sync(
|
|
424
448
|
|
425
449
|
|
426
450
|
def get_sync_time(
|
427
|
-
|
428
|
-
|
429
|
-
|
430
|
-
|
431
|
-
|
432
|
-
|
451
|
+
self,
|
452
|
+
params: Optional[Dict[str, Any]] = None,
|
453
|
+
newest: bool = True,
|
454
|
+
apply_backtrack_interval: bool = False,
|
455
|
+
round_down: bool = False,
|
456
|
+
debug: bool = False
|
457
|
+
) -> Union['datetime', None]:
|
433
458
|
"""
|
434
459
|
Get the most recent datetime value for a Pipe.
|
435
460
|
|
@@ -443,6 +468,9 @@ def get_sync_time(
|
|
443
468
|
If `True`, get the most recent datetime (honoring `params`).
|
444
469
|
If `False`, get the oldest datetime (`ASC` instead of `DESC`).
|
445
470
|
|
471
|
+
apply_backtrack_interval: bool, default False
|
472
|
+
If `True`, subtract the backtrack interval from the sync time.
|
473
|
+
|
446
474
|
round_down: bool, default False
|
447
475
|
If `True`, round down the datetime value to the nearest minute.
|
448
476
|
|
@@ -461,15 +489,22 @@ def get_sync_time(
|
|
461
489
|
with Venv(get_connector_plugin(self.instance_connector)):
|
462
490
|
sync_time = self.instance_connector.get_sync_time(
|
463
491
|
self,
|
464
|
-
params
|
465
|
-
newest
|
466
|
-
debug
|
492
|
+
params=params,
|
493
|
+
newest=newest,
|
494
|
+
debug=debug,
|
467
495
|
)
|
468
496
|
|
469
|
-
if
|
470
|
-
|
497
|
+
if round_down and isinstance(sync_time, datetime):
|
498
|
+
sync_time = round_time(sync_time, timedelta(minutes=1))
|
499
|
+
|
500
|
+
if apply_backtrack_interval and sync_time is not None:
|
501
|
+
backtrack_interval = self.get_backtrack_interval(debug=debug)
|
502
|
+
try:
|
503
|
+
sync_time -= backtrack_interval
|
504
|
+
except Exception as e:
|
505
|
+
warn(f"Failed to apply backtrack interval:\n{e}")
|
471
506
|
|
472
|
-
return
|
507
|
+
return sync_time
|
473
508
|
|
474
509
|
|
475
510
|
def exists(
|
meerschaum/plugins/__init__.py
CHANGED
@@ -27,7 +27,7 @@ _locks = {
|
|
27
27
|
'PLUGINS_INTERNAL_LOCK_PATH': RLock(),
|
28
28
|
}
|
29
29
|
__all__ = (
|
30
|
-
"Plugin", "make_action", "api_plugin", "import_plugins",
|
30
|
+
"Plugin", "make_action", "api_plugin", "dash_plugin", "import_plugins",
|
31
31
|
"reload_plugins", "get_plugins", "get_data_plugins", "add_plugin_argument",
|
32
32
|
"pre_sync_hook", "post_sync_hook",
|
33
33
|
)
|
@@ -0,0 +1,333 @@
|
|
1
|
+
#! /usr/bin/env python3
|
2
|
+
# -*- coding: utf-8 -*-
|
3
|
+
# vim:fenc=utf-8
|
4
|
+
|
5
|
+
"""
|
6
|
+
Define the bootstrapping wizard for creating plugins.
|
7
|
+
"""
|
8
|
+
|
9
|
+
import re
|
10
|
+
import pathlib
|
11
|
+
import meerschaum as mrsm
|
12
|
+
from meerschaum.utils.typing import Any, SuccessTuple, Dict, List
|
13
|
+
from meerschaum.utils.warnings import warn, info
|
14
|
+
from meerschaum.utils.prompt import prompt, choose, yes_no
|
15
|
+
from meerschaum.utils.formatting._shell import clear_screen
|
16
|
+
|
17
|
+
FEATURE_CHOICES: Dict[str, str] = {
|
18
|
+
'fetch' : 'Fetch data\n (e.g. extracting from a remote API)\n',
|
19
|
+
'connector': 'Custom connector\n (e.g. manage credentials)\n',
|
20
|
+
'action' : 'New actions\n (e.g. `mrsm sing song`)\n',
|
21
|
+
'api' : 'New API endpoints\n (e.g. `POST /my/new/endpoint`)\n',
|
22
|
+
'web' : 'New web console page\n (e.g. `/dash/my-web-app`)\n',
|
23
|
+
}
|
24
|
+
|
25
|
+
IMPORTS_LINES: Dict[str, str] = {
|
26
|
+
'stdlib': (
|
27
|
+
"from datetime import datetime, timedelta, timezone\n"
|
28
|
+
),
|
29
|
+
'default': (
|
30
|
+
"import meerschaum as mrsm\n"
|
31
|
+
"from meerschaum.config import get_plugin_config, write_plugin_config\n"
|
32
|
+
),
|
33
|
+
'connector': (
|
34
|
+
"from meerschaum.connectors import Connector, make_connector\n"
|
35
|
+
),
|
36
|
+
'action': (
|
37
|
+
"from meerschaum.actions import make_action\n"
|
38
|
+
),
|
39
|
+
'api': (
|
40
|
+
"from meerschaum.plugins import api_plugin\n"
|
41
|
+
),
|
42
|
+
'web': (
|
43
|
+
"from meerschaum.plugins import web_page, dash_plugin\n"
|
44
|
+
),
|
45
|
+
'api+web': (
|
46
|
+
"from meerschaum.plugins import api_plugin, web_page, dash_plugin\n"
|
47
|
+
),
|
48
|
+
}
|
49
|
+
|
50
|
+
### TODO: Add feature for custom connectors.
|
51
|
+
FEATURE_LINES: Dict[str, str] = {
|
52
|
+
'header': (
|
53
|
+
"#! /usr/bin/env python3\n"
|
54
|
+
"# -*- coding: utf-8 -*-\n\n"
|
55
|
+
"\"\"\"\n"
|
56
|
+
"Implement the plugin '{plugin_name}'.\n\n"
|
57
|
+
"See the Writing Plugins guide for more information:\n"
|
58
|
+
"https://meerschaum.io/reference/plugins/writing-plugins/\n"
|
59
|
+
"\"\"\"\n\n"
|
60
|
+
),
|
61
|
+
'default': (
|
62
|
+
"__version__ = '0.0.1'\n"
|
63
|
+
"\n# Add any dependencies to `required` (similar to `requirements.txt`).\n"
|
64
|
+
"required: list[str] = []\n\n\n"
|
65
|
+
),
|
66
|
+
'setup': (
|
67
|
+
"def setup(**kwargs) -> mrsm.SuccessTuple:\n"
|
68
|
+
" \"\"\"Executed during installation and `mrsm setup plugin {plugin_name}`.\"\"\"\n"
|
69
|
+
" return True, \"Success\"\n\n\n"
|
70
|
+
),
|
71
|
+
'register': (
|
72
|
+
"def register(pipe: mrsm.Pipe):\n"
|
73
|
+
" \"\"\"Return the default parameters for a new pipe.\"\"\"\n"
|
74
|
+
" return {{\n"
|
75
|
+
" 'columns': {{\n"
|
76
|
+
" 'datetime': {dt_col_name},\n"
|
77
|
+
" }}\n"
|
78
|
+
" }}\n\n\n"
|
79
|
+
),
|
80
|
+
'fetch': (
|
81
|
+
"def fetch(\n"
|
82
|
+
" pipe: mrsm.Pipe,\n"
|
83
|
+
" begin: datetime | None = None,\n"
|
84
|
+
" end: datetime | None = None,\n"
|
85
|
+
" **kwargs\n"
|
86
|
+
"):\n"
|
87
|
+
" \"\"\"Return or yield dataframes.\"\"\"\n"
|
88
|
+
" docs = []\n"
|
89
|
+
" # populate docs with dictionaries (rows).\n"
|
90
|
+
" return docs\n\n\n"
|
91
|
+
),
|
92
|
+
'connector': (
|
93
|
+
"@make_connector\n"
|
94
|
+
"class {plugin_name_capitalized}Connector(Connector):\n"
|
95
|
+
" \"\"\"Implement '{plugin_name_lower}' connectors.\"\"\"\n\n"
|
96
|
+
" REQUIRED_ATTRIBUTES: list[str] = []\n"
|
97
|
+
" \n"
|
98
|
+
" def fetch(\n"
|
99
|
+
" self,\n"
|
100
|
+
" pipe: mrsm.Pipe,\n"
|
101
|
+
" begin: datetime | None = None,\n"
|
102
|
+
" end: datetime | None = None,\n"
|
103
|
+
" **kwargs\n"
|
104
|
+
" ):\n"
|
105
|
+
" \"\"\"Return or yield dataframes.\"\"\"\n"
|
106
|
+
" docs = []\n"
|
107
|
+
" # populate docs with dictionaries (rows).\n"
|
108
|
+
" return docs\n\n\n"
|
109
|
+
),
|
110
|
+
'action': (
|
111
|
+
"@make_action\n"
|
112
|
+
"def {action_name}(**kwargs) -> mrsm.SuccessTuple:\n"
|
113
|
+
" \"\"\"Run `mrsm {action_spaces}` to trigger.\"\"\"\n"
|
114
|
+
" return True, \"Success\"\n\n\n"
|
115
|
+
),
|
116
|
+
'api': (
|
117
|
+
"@api_plugin\n"
|
118
|
+
"def init_app(fastapi_app):\n"
|
119
|
+
" \"\"\"Add new endpoints to the FastAPI app.\"\"\"\n\n"
|
120
|
+
" import fastapi\n"
|
121
|
+
" from meerschaum.api import manager\n\n"
|
122
|
+
" @fastapi_app.get('/{plugin_name}')\n"
|
123
|
+
" def get_my_endpoint(curr_user=fastapi.Depends(manager)):\n"
|
124
|
+
" return {{'message': \"Hello from plugin '{plugin_name}'!\"}}\n\n\n"
|
125
|
+
),
|
126
|
+
'web': (
|
127
|
+
"@dash_plugin\n"
|
128
|
+
"def init_dash(dash_app):\n"
|
129
|
+
" \"\"\"Initialize the Plotly Dash application.\"\"\"\n\n"
|
130
|
+
" import dash.html as html\n"
|
131
|
+
" import dash.dcc as dcc\n"
|
132
|
+
" from dash import Input, Output, State, no_update\n"
|
133
|
+
" import dash_bootstrap_components as dbc\n\n"
|
134
|
+
" # Create a new page at the path `/dash/{plugin_name}`.\n"
|
135
|
+
" @web_page('{plugin_name}', login_required=False)\n"
|
136
|
+
" def page_layout():\n"
|
137
|
+
" \"\"\"Return the layout objects for this page.\"\"\"\n"
|
138
|
+
" return dbc.Container([\n"
|
139
|
+
" dcc.Location(id='{plugin_name}-location'),\n"
|
140
|
+
" html.Div(id='output-div'),\n"
|
141
|
+
" ])\n\n"
|
142
|
+
" @dash_app.callback(\n"
|
143
|
+
" Output('output-div', 'children'),\n"
|
144
|
+
" Input('{plugin_name}-location', 'pathname'),\n"
|
145
|
+
" )\n"
|
146
|
+
" def render_page_on_url_change(pathname: str):\n"
|
147
|
+
" \"\"\"Reload page contents when the URL path changes.\"\"\"\n"
|
148
|
+
" return html.H1(\"Hello from plugin '{plugin_name}'!\")\n\n\n"
|
149
|
+
),
|
150
|
+
}
|
151
|
+
|
152
|
+
def bootstrap_plugin(
|
153
|
+
plugin_name: str,
|
154
|
+
debug: bool = False,
|
155
|
+
**kwargs: Any
|
156
|
+
) -> SuccessTuple:
|
157
|
+
"""
|
158
|
+
Prompt the user for features and create a plugin file.
|
159
|
+
"""
|
160
|
+
from meerschaum.utils.misc import edit_file
|
161
|
+
plugins_dir_path = _get_plugins_dir_path()
|
162
|
+
clear_screen(debug=debug)
|
163
|
+
info(
|
164
|
+
"Answer the questions below to pick out features.\n"
|
165
|
+
+ " See the Writing Plugins guide for documentation:\n"
|
166
|
+
+ " https://meerschaum.io/reference/plugins/writing-plugins/ "
|
167
|
+
+ "for documentation.\n"
|
168
|
+
)
|
169
|
+
|
170
|
+
plugin = mrsm.Plugin(plugin_name)
|
171
|
+
if plugin.is_installed():
|
172
|
+
uninstall_success, uninstall_msg = _ask_to_uninstall(plugin)
|
173
|
+
if not uninstall_success:
|
174
|
+
return uninstall_success, uninstall_msg
|
175
|
+
clear_screen(debug=debug)
|
176
|
+
|
177
|
+
features: List[str] = choose(
|
178
|
+
"Which of the following features would you like to add to your plugin?",
|
179
|
+
list(FEATURE_CHOICES.items()),
|
180
|
+
default = 'fetch',
|
181
|
+
multiple = True,
|
182
|
+
as_indices = True,
|
183
|
+
**kwargs
|
184
|
+
)
|
185
|
+
|
186
|
+
clear_screen(debug=debug)
|
187
|
+
|
188
|
+
action_name = ''
|
189
|
+
if 'action' in features:
|
190
|
+
action_name = _get_action_name()
|
191
|
+
clear_screen(debug=debug)
|
192
|
+
action_spaces = action_name.replace('_', ' ')
|
193
|
+
|
194
|
+
dt_col_name = None
|
195
|
+
if 'fetch' in features:
|
196
|
+
dt_col_name = _get_quoted_dt_col_name()
|
197
|
+
clear_screen(debug=debug)
|
198
|
+
|
199
|
+
plugin_labels = {
|
200
|
+
'plugin_name': plugin_name,
|
201
|
+
'plugin_name_capitalized': re.split(
|
202
|
+
r'[-_]', plugin_name.lower()
|
203
|
+
)[0].capitalize(),
|
204
|
+
'plugin_name_lower': plugin_name.lower(),
|
205
|
+
'action_name': action_name,
|
206
|
+
'action_spaces': action_spaces,
|
207
|
+
'dt_col_name': dt_col_name,
|
208
|
+
}
|
209
|
+
|
210
|
+
body_text = ""
|
211
|
+
body_text += FEATURE_LINES['header'].format(**plugin_labels)
|
212
|
+
body_text += IMPORTS_LINES['stdlib'].format(**plugin_labels)
|
213
|
+
body_text += IMPORTS_LINES['default'].format(**plugin_labels)
|
214
|
+
if 'connector' in features:
|
215
|
+
body_text += IMPORTS_LINES['connector'].format(**plugin_labels)
|
216
|
+
if 'action' in features:
|
217
|
+
body_text += IMPORTS_LINES['action'].format(**plugin_labels)
|
218
|
+
if 'api' in features and 'web' in features:
|
219
|
+
body_text += IMPORTS_LINES['api+web'].format(**plugin_labels)
|
220
|
+
elif 'api' in features:
|
221
|
+
body_text += IMPORTS_LINES['api'].format(**plugin_labels)
|
222
|
+
elif 'web' in features:
|
223
|
+
body_text += IMPORTS_LINES['web'].format(**plugin_labels)
|
224
|
+
|
225
|
+
body_text += "\n"
|
226
|
+
body_text += FEATURE_LINES['default'].format(**plugin_labels)
|
227
|
+
body_text += FEATURE_LINES['setup'].format(**plugin_labels)
|
228
|
+
|
229
|
+
if 'fetch' in features:
|
230
|
+
body_text += FEATURE_LINES['register'].format(**plugin_labels)
|
231
|
+
body_text += FEATURE_LINES['fetch'].format(**plugin_labels)
|
232
|
+
|
233
|
+
if 'connector' in features:
|
234
|
+
body_text += FEATURE_LINES['connector'].format(**plugin_labels)
|
235
|
+
|
236
|
+
if 'action' in features:
|
237
|
+
body_text += FEATURE_LINES['action'].format(**plugin_labels)
|
238
|
+
|
239
|
+
if 'api' in features:
|
240
|
+
body_text += FEATURE_LINES['api'].format(**plugin_labels)
|
241
|
+
|
242
|
+
if 'web' in features:
|
243
|
+
body_text += FEATURE_LINES['web'].format(**plugin_labels)
|
244
|
+
|
245
|
+
try:
|
246
|
+
plugin_path = plugins_dir_path / (plugin_name + '.py')
|
247
|
+
with open(plugin_path, 'w+', encoding='utf-8') as f:
|
248
|
+
f.write(body_text.rstrip())
|
249
|
+
except Exception as e:
|
250
|
+
error_msg = f"Failed to write file '{plugin_path}':\n{e}"
|
251
|
+
return False, error_msg
|
252
|
+
|
253
|
+
clear_screen(debug=debug)
|
254
|
+
mrsm.pprint((True, f"Successfully created file '{plugin_path}'."))
|
255
|
+
try:
|
256
|
+
_ = prompt(
|
257
|
+
f"Press [Enter] to edit plugin '{plugin_name}',"
|
258
|
+
+ " [CTRL+C] to skip.",
|
259
|
+
icon = False,
|
260
|
+
)
|
261
|
+
except (KeyboardInterrupt, Exception):
|
262
|
+
return True, "Success"
|
263
|
+
|
264
|
+
edit_file(plugin_path, debug=debug)
|
265
|
+
return True, "Success"
|
266
|
+
|
267
|
+
|
268
|
+
def _get_plugins_dir_path() -> pathlib.Path:
|
269
|
+
from meerschaum.config.paths import PLUGINS_DIR_PATHS
|
270
|
+
|
271
|
+
if not PLUGINS_DIR_PATHS:
|
272
|
+
raise EnvironmentError("No plugin dir path could be found.")
|
273
|
+
|
274
|
+
if len(PLUGINS_DIR_PATHS) == 1:
|
275
|
+
return PLUGINS_DIR_PATHS[0]
|
276
|
+
|
277
|
+
return pathlib.Path(
|
278
|
+
choose(
|
279
|
+
"In which directory do you want to write your plugin?",
|
280
|
+
[path.as_posix() for path in PLUGINS_DIR_PATHS],
|
281
|
+
numeric = True,
|
282
|
+
multiple = False,
|
283
|
+
default = PLUGINS_DIR_PATHS[0].as_posix(),
|
284
|
+
)
|
285
|
+
)
|
286
|
+
|
287
|
+
|
288
|
+
def _ask_to_uninstall(plugin: mrsm.Plugin, **kwargs: Any) -> SuccessTuple:
|
289
|
+
from meerschaum._internal.entry import entry
|
290
|
+
warn(f"Plugin '{plugin}' is already installed!", stack=False)
|
291
|
+
uninstall_plugin = yes_no(
|
292
|
+
f"Do you want to first uninstall '{plugin}'?",
|
293
|
+
default = 'n',
|
294
|
+
**kwargs
|
295
|
+
)
|
296
|
+
if not uninstall_plugin:
|
297
|
+
return False, f"Plugin '{plugin}' already exists."
|
298
|
+
|
299
|
+
return entry(['uninstall', 'plugin', plugin.name, '-f'])
|
300
|
+
|
301
|
+
|
302
|
+
def _get_action_name() -> str:
|
303
|
+
while True:
|
304
|
+
try:
|
305
|
+
action_name = prompt(
|
306
|
+
"What is name of your action?\n "
|
307
|
+
+ "(separate subactions with spaces, e.g. `sing song`):"
|
308
|
+
).replace(' ', '_')
|
309
|
+
except KeyboardInterrupt:
|
310
|
+
return False, "Aborted plugin creation."
|
311
|
+
|
312
|
+
if action_name:
|
313
|
+
break
|
314
|
+
warn("Please enter an action.", stack=False)
|
315
|
+
return action_name
|
316
|
+
|
317
|
+
|
318
|
+
def _get_quoted_dt_col_name() -> str:
|
319
|
+
try:
|
320
|
+
dt_col_name = prompt(
|
321
|
+
"Enter the datetime column name ([CTRL+C] to skip):"
|
322
|
+
)
|
323
|
+
except (Exception, KeyboardInterrupt):
|
324
|
+
dt_col_name = None
|
325
|
+
|
326
|
+
if dt_col_name is None:
|
327
|
+
dt_col_name = 'None'
|
328
|
+
elif '"' in dt_col_name or "'" in dt_col_name:
|
329
|
+
dt_col_name = f"\"\"\"{dt_col_name}\"\"\""
|
330
|
+
else:
|
331
|
+
dt_col_name = f"\"{dt_col_name}\""
|
332
|
+
|
333
|
+
return dt_col_name
|
@@ -185,7 +185,7 @@ class Daemon:
|
|
185
185
|
result = self.target(*self.target_args, **self.target_kw)
|
186
186
|
self.properties['result'] = result
|
187
187
|
except Exception as e:
|
188
|
-
warn(e, stacklevel=3)
|
188
|
+
warn(f"Exception in daemon target function: {e}", stacklevel=3)
|
189
189
|
result = e
|
190
190
|
finally:
|
191
191
|
self._log_refresh_timer.cancel()
|
@@ -203,9 +203,20 @@ class Daemon:
|
|
203
203
|
daemon_error = traceback.format_exc()
|
204
204
|
with open(DAEMON_ERROR_LOG_PATH, 'a+', encoding='utf-8') as f:
|
205
205
|
f.write(daemon_error)
|
206
|
+
warn(f"Encountered an error while running the daemon '{self}':\n{daemon_error}")
|
207
|
+
finally:
|
208
|
+
self._cleanup_on_exit()
|
206
209
|
|
207
|
-
|
208
|
-
|
210
|
+
def _cleanup_on_exit(self):
|
211
|
+
"""Perform cleanup operations when the daemon exits."""
|
212
|
+
try:
|
213
|
+
self._log_refresh_timer.cancel()
|
214
|
+
self.rotating_log.close()
|
215
|
+
if self.pid is None and self.pid_path.exists():
|
216
|
+
self.pid_path.unlink()
|
217
|
+
self._capture_process_timestamp('ended')
|
218
|
+
except Exception as e:
|
219
|
+
warn(f"Error during daemon cleanup: {e}")
|
209
220
|
|
210
221
|
|
211
222
|
def _capture_process_timestamp(
|