meerschaum 2.9.4__py3-none-any.whl → 3.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/__init__.py +5 -2
- meerschaum/_internal/__init__.py +1 -0
- meerschaum/_internal/arguments/_parse_arguments.py +4 -4
- meerschaum/_internal/arguments/_parser.py +33 -4
- meerschaum/_internal/cli/__init__.py +6 -0
- meerschaum/_internal/cli/daemons.py +103 -0
- meerschaum/_internal/cli/entry.py +220 -0
- meerschaum/_internal/cli/workers.py +435 -0
- meerschaum/_internal/docs/index.py +48 -2
- meerschaum/_internal/entry.py +50 -14
- meerschaum/_internal/shell/Shell.py +121 -29
- meerschaum/_internal/shell/__init__.py +4 -1
- meerschaum/_internal/static.py +359 -0
- meerschaum/_internal/term/TermPageHandler.py +1 -2
- meerschaum/_internal/term/__init__.py +40 -6
- meerschaum/_internal/term/tools.py +33 -8
- meerschaum/actions/__init__.py +6 -4
- meerschaum/actions/api.py +53 -13
- meerschaum/actions/attach.py +1 -0
- meerschaum/actions/bootstrap.py +8 -8
- meerschaum/actions/delete.py +4 -2
- meerschaum/actions/edit.py +171 -25
- meerschaum/actions/login.py +8 -8
- meerschaum/actions/register.py +143 -6
- meerschaum/actions/reload.py +22 -5
- meerschaum/actions/restart.py +14 -0
- meerschaum/actions/show.py +184 -31
- meerschaum/actions/start.py +166 -17
- meerschaum/actions/stop.py +38 -2
- meerschaum/actions/sync.py +7 -2
- meerschaum/actions/tag.py +9 -8
- meerschaum/actions/verify.py +5 -8
- meerschaum/api/__init__.py +45 -15
- meerschaum/api/_events.py +46 -4
- meerschaum/api/_oauth2.py +162 -9
- meerschaum/api/_tokens.py +102 -0
- meerschaum/api/dash/__init__.py +0 -3
- meerschaum/api/dash/callbacks/__init__.py +1 -0
- meerschaum/api/dash/callbacks/custom.py +4 -3
- meerschaum/api/dash/callbacks/dashboard.py +228 -117
- meerschaum/api/dash/callbacks/jobs.py +14 -7
- meerschaum/api/dash/callbacks/login.py +10 -1
- meerschaum/api/dash/callbacks/pipes.py +194 -14
- meerschaum/api/dash/callbacks/plugins.py +0 -1
- meerschaum/api/dash/callbacks/register.py +10 -3
- meerschaum/api/dash/callbacks/settings/password_reset.py +2 -2
- meerschaum/api/dash/callbacks/tokens.py +389 -0
- meerschaum/api/dash/components.py +36 -15
- meerschaum/api/dash/jobs.py +1 -1
- meerschaum/api/dash/keys.py +35 -93
- meerschaum/api/dash/pages/__init__.py +2 -1
- meerschaum/api/dash/pages/dashboard.py +1 -20
- meerschaum/api/dash/pages/{job.py → jobs.py} +10 -7
- meerschaum/api/dash/pages/login.py +2 -2
- meerschaum/api/dash/pages/pipes.py +16 -5
- meerschaum/api/dash/pages/settings/password_reset.py +1 -1
- meerschaum/api/dash/pages/tokens.py +53 -0
- meerschaum/api/dash/pipes.py +438 -88
- meerschaum/api/dash/sessions.py +12 -0
- meerschaum/api/dash/tokens.py +603 -0
- meerschaum/api/dash/websockets.py +1 -1
- meerschaum/api/dash/webterm.py +18 -6
- meerschaum/api/models/__init__.py +23 -3
- meerschaum/api/models/_actions.py +22 -0
- meerschaum/api/models/_pipes.py +91 -7
- meerschaum/api/models/_tokens.py +81 -0
- meerschaum/api/resources/static/css/dash.css +16 -0
- meerschaum/api/resources/static/js/terminado.js +3 -0
- meerschaum/api/resources/static/js/xterm-addon-unicode11.js +2 -0
- meerschaum/api/resources/templates/termpage.html +13 -0
- meerschaum/api/routes/__init__.py +1 -0
- meerschaum/api/routes/_actions.py +3 -4
- meerschaum/api/routes/_connectors.py +3 -7
- meerschaum/api/routes/_jobs.py +26 -35
- meerschaum/api/routes/_login.py +120 -15
- meerschaum/api/routes/_misc.py +5 -10
- meerschaum/api/routes/_pipes.py +178 -143
- meerschaum/api/routes/_plugins.py +38 -28
- meerschaum/api/routes/_tokens.py +236 -0
- meerschaum/api/routes/_users.py +47 -35
- meerschaum/api/routes/_version.py +3 -3
- meerschaum/api/routes/_webterm.py +3 -3
- meerschaum/config/__init__.py +100 -30
- meerschaum/config/_default.py +132 -64
- meerschaum/config/_edit.py +38 -32
- meerschaum/config/_formatting.py +2 -0
- meerschaum/config/_patch.py +10 -8
- meerschaum/config/_paths.py +133 -13
- meerschaum/config/_read_config.py +87 -36
- meerschaum/config/_sync.py +6 -3
- meerschaum/config/_version.py +1 -1
- meerschaum/config/environment.py +262 -0
- meerschaum/config/stack/__init__.py +37 -15
- meerschaum/config/static.py +18 -0
- meerschaum/connectors/_Connector.py +11 -6
- meerschaum/connectors/__init__.py +41 -22
- meerschaum/connectors/api/_APIConnector.py +34 -6
- meerschaum/connectors/api/_actions.py +2 -2
- meerschaum/connectors/api/_jobs.py +12 -1
- meerschaum/connectors/api/_login.py +33 -7
- meerschaum/connectors/api/_misc.py +2 -2
- meerschaum/connectors/api/_pipes.py +23 -32
- meerschaum/connectors/api/_plugins.py +2 -2
- meerschaum/connectors/api/_request.py +1 -1
- meerschaum/connectors/api/_tokens.py +146 -0
- meerschaum/connectors/api/_users.py +70 -58
- meerschaum/connectors/instance/_InstanceConnector.py +83 -0
- meerschaum/connectors/instance/__init__.py +10 -0
- meerschaum/connectors/instance/_pipes.py +442 -0
- meerschaum/connectors/instance/_plugins.py +159 -0
- meerschaum/connectors/instance/_tokens.py +317 -0
- meerschaum/connectors/instance/_users.py +188 -0
- meerschaum/connectors/parse.py +5 -2
- meerschaum/connectors/sql/_SQLConnector.py +22 -5
- meerschaum/connectors/sql/_cli.py +12 -11
- meerschaum/connectors/sql/_create_engine.py +12 -168
- meerschaum/connectors/sql/_fetch.py +2 -18
- meerschaum/connectors/sql/_pipes.py +295 -278
- meerschaum/connectors/sql/_plugins.py +29 -0
- meerschaum/connectors/sql/_sql.py +47 -22
- meerschaum/connectors/sql/_users.py +36 -2
- meerschaum/connectors/sql/tables/__init__.py +254 -122
- meerschaum/connectors/valkey/_ValkeyConnector.py +5 -7
- meerschaum/connectors/valkey/_pipes.py +60 -31
- meerschaum/connectors/valkey/_plugins.py +2 -26
- meerschaum/core/Pipe/__init__.py +115 -85
- meerschaum/core/Pipe/_attributes.py +425 -124
- meerschaum/core/Pipe/_bootstrap.py +54 -24
- meerschaum/core/Pipe/_cache.py +555 -0
- meerschaum/core/Pipe/_clear.py +0 -11
- meerschaum/core/Pipe/_data.py +96 -68
- meerschaum/core/Pipe/_deduplicate.py +0 -13
- meerschaum/core/Pipe/_delete.py +12 -21
- meerschaum/core/Pipe/_drop.py +11 -23
- meerschaum/core/Pipe/_dtypes.py +49 -19
- meerschaum/core/Pipe/_edit.py +14 -4
- meerschaum/core/Pipe/_fetch.py +1 -1
- meerschaum/core/Pipe/_index.py +8 -14
- meerschaum/core/Pipe/_show.py +5 -5
- meerschaum/core/Pipe/_sync.py +123 -204
- meerschaum/core/Pipe/_verify.py +4 -4
- meerschaum/{plugins → core/Plugin}/_Plugin.py +16 -12
- meerschaum/core/Plugin/__init__.py +1 -1
- meerschaum/core/Token/_Token.py +220 -0
- meerschaum/core/Token/__init__.py +12 -0
- meerschaum/core/User/_User.py +35 -10
- meerschaum/core/User/__init__.py +9 -1
- meerschaum/core/__init__.py +1 -0
- meerschaum/jobs/_Executor.py +88 -4
- meerschaum/jobs/_Job.py +149 -38
- meerschaum/jobs/__init__.py +3 -2
- meerschaum/jobs/systemd.py +8 -3
- meerschaum/models/__init__.py +35 -0
- meerschaum/models/pipes.py +247 -0
- meerschaum/models/tokens.py +38 -0
- meerschaum/models/users.py +26 -0
- meerschaum/plugins/__init__.py +301 -88
- meerschaum/plugins/bootstrap.py +510 -4
- meerschaum/utils/_get_pipes.py +97 -30
- meerschaum/utils/daemon/Daemon.py +199 -43
- meerschaum/utils/daemon/FileDescriptorInterceptor.py +0 -1
- meerschaum/utils/daemon/RotatingFile.py +63 -36
- meerschaum/utils/daemon/StdinFile.py +53 -13
- meerschaum/utils/daemon/__init__.py +47 -6
- meerschaum/utils/daemon/_names.py +6 -3
- meerschaum/utils/dataframe.py +480 -82
- meerschaum/utils/debug.py +49 -19
- meerschaum/utils/dtypes/__init__.py +478 -37
- meerschaum/utils/dtypes/sql.py +369 -29
- meerschaum/utils/formatting/__init__.py +5 -2
- meerschaum/utils/formatting/_jobs.py +1 -1
- meerschaum/utils/formatting/_pipes.py +52 -50
- meerschaum/utils/formatting/_pprint.py +1 -0
- meerschaum/utils/formatting/_shell.py +44 -18
- meerschaum/utils/misc.py +268 -186
- meerschaum/utils/packages/__init__.py +25 -40
- meerschaum/utils/packages/_packages.py +42 -34
- meerschaum/utils/pipes.py +213 -0
- meerschaum/utils/process.py +2 -2
- meerschaum/utils/prompt.py +175 -144
- meerschaum/utils/schedule.py +2 -1
- meerschaum/utils/sql.py +135 -49
- meerschaum/utils/threading.py +42 -0
- meerschaum/utils/typing.py +1 -4
- meerschaum/utils/venv/_Venv.py +2 -2
- meerschaum/utils/venv/__init__.py +7 -7
- meerschaum/utils/warnings.py +19 -13
- {meerschaum-2.9.4.dist-info → meerschaum-3.0.0.dist-info}/METADATA +94 -96
- meerschaum-3.0.0.dist-info/RECORD +289 -0
- {meerschaum-2.9.4.dist-info → meerschaum-3.0.0.dist-info}/WHEEL +1 -1
- meerschaum-3.0.0.dist-info/licenses/NOTICE +2 -0
- meerschaum/api/models/_interfaces.py +0 -15
- meerschaum/api/models/_locations.py +0 -15
- meerschaum/api/models/_metrics.py +0 -15
- meerschaum/config/_environment.py +0 -145
- meerschaum/config/static/__init__.py +0 -186
- meerschaum-2.9.4.dist-info/RECORD +0 -263
- {meerschaum-2.9.4.dist-info → meerschaum-3.0.0.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.9.4.dist-info → meerschaum-3.0.0.dist-info}/licenses/LICENSE +0 -0
- {meerschaum-2.9.4.dist-info → meerschaum-3.0.0.dist-info}/top_level.txt +0 -0
- {meerschaum-2.9.4.dist-info → meerschaum-3.0.0.dist-info}/zip-safe +0 -0
meerschaum/plugins/__init__.py
CHANGED
@@ -7,20 +7,25 @@ Expose plugin management APIs from the `meerschaum.plugins` module.
|
|
7
7
|
"""
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
|
+
|
11
|
+
import pathlib
|
10
12
|
import functools
|
13
|
+
|
11
14
|
import meerschaum as mrsm
|
12
15
|
from meerschaum.utils.typing import Callable, Any, Union, Optional, Dict, List, Tuple
|
13
16
|
from meerschaum.utils.threading import Lock, RLock
|
14
|
-
from meerschaum.
|
17
|
+
from meerschaum.core.Plugin import Plugin
|
15
18
|
|
16
19
|
_api_plugins: Dict[str, List[Callable[['fastapi.App'], Any]]] = {}
|
17
|
-
_pre_sync_hooks: Dict[str, List[Callable[[Any], Any]]] = {}
|
18
|
-
_post_sync_hooks: Dict[str, List[Callable[[Any], Any]]] = {}
|
20
|
+
_pre_sync_hooks: Dict[Union[str, None], List[Callable[[Any], Any]]] = {}
|
21
|
+
_post_sync_hooks: Dict[Union[str, None], List[Callable[[Any], Any]]] = {}
|
22
|
+
_actions_daemon_enabled: Dict[str, bool] = {}
|
19
23
|
_locks = {
|
20
24
|
'_api_plugins': RLock(),
|
21
25
|
'_dash_plugins': RLock(),
|
22
26
|
'_pre_sync_hooks': RLock(),
|
23
27
|
'_post_sync_hooks': RLock(),
|
28
|
+
'_actions_daemon_enabled': RLock(),
|
24
29
|
'__path__': RLock(),
|
25
30
|
'sys.path': RLock(),
|
26
31
|
'internal_plugins': RLock(),
|
@@ -28,21 +33,37 @@ _locks = {
|
|
28
33
|
'PLUGINS_INTERNAL_LOCK_PATH': RLock(),
|
29
34
|
}
|
30
35
|
__all__ = (
|
31
|
-
"Plugin",
|
32
|
-
"
|
33
|
-
"
|
34
|
-
"
|
36
|
+
"Plugin",
|
37
|
+
"make_action",
|
38
|
+
"api_plugin",
|
39
|
+
"dash_plugin",
|
40
|
+
"web_page",
|
41
|
+
"import_plugins",
|
42
|
+
"from_plugin_import",
|
43
|
+
"reload_plugins",
|
44
|
+
"get_plugins",
|
45
|
+
"get_data_plugins",
|
46
|
+
"add_plugin_argument",
|
47
|
+
"pre_sync_hook",
|
48
|
+
"post_sync_hook",
|
35
49
|
)
|
36
50
|
__pdoc__ = {
|
37
|
-
'venvs': False,
|
51
|
+
'venvs': False,
|
52
|
+
'data': False,
|
53
|
+
'stack': False,
|
54
|
+
'plugins': False,
|
38
55
|
}
|
39
56
|
|
57
|
+
|
40
58
|
def make_action(
|
41
|
-
function: Callable[[Any], Any],
|
59
|
+
function: Optional[Callable[[Any], Any]] = None,
|
42
60
|
shell: bool = False,
|
43
61
|
activate: bool = True,
|
44
62
|
deactivate: bool = True,
|
45
|
-
debug: bool = False
|
63
|
+
debug: bool = False,
|
64
|
+
daemon: bool = True,
|
65
|
+
skip_if_loaded: bool = True,
|
66
|
+
_plugin_name: Optional[str] = None,
|
46
67
|
) -> Callable[[Any], Any]:
|
47
68
|
"""
|
48
69
|
Make a function a Meerschaum action. Useful for plugins that are adding multiple actions.
|
@@ -69,25 +90,36 @@ def make_action(
|
|
69
90
|
... return True, "Success"
|
70
91
|
>>>
|
71
92
|
"""
|
93
|
+
def _decorator(func: Callable[[Any], Any]) -> Callable[[Any], Any]:
|
94
|
+
from meerschaum.actions import actions, _custom_actions_plugins, _plugins_actions
|
95
|
+
if skip_if_loaded and func.__name__ in actions:
|
96
|
+
return func
|
72
97
|
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
plugin_name = (
|
77
|
-
package_name.split('.')[1]
|
78
|
-
if package_name.startswith('plugins.') else None
|
79
|
-
)
|
80
|
-
plugin = Plugin(plugin_name) if plugin_name else None
|
81
|
-
|
82
|
-
if debug:
|
83
|
-
from meerschaum.utils.debug import dprint
|
84
|
-
dprint(
|
85
|
-
f"Adding action '{function.__name__}' from plugin " +
|
86
|
-
f"'{plugin}'..."
|
98
|
+
from meerschaum.config.paths import PLUGINS_RESOURCES_PATH
|
99
|
+
plugin_name = (
|
100
|
+
func.__name__.split(f"{PLUGINS_RESOURCES_PATH.stem}.", maxsplit=1)[-1].split('.')[0]
|
87
101
|
)
|
102
|
+
plugin = Plugin(plugin_name) if plugin_name else None
|
88
103
|
|
89
|
-
|
90
|
-
|
104
|
+
if debug:
|
105
|
+
from meerschaum.utils.debug import dprint
|
106
|
+
dprint(
|
107
|
+
f"Adding action '{func.__name__}' from plugin "
|
108
|
+
f"'{plugin}'..."
|
109
|
+
)
|
110
|
+
|
111
|
+
actions[func.__name__] = func
|
112
|
+
_custom_actions_plugins[func.__name__] = plugin_name
|
113
|
+
if plugin_name not in _plugins_actions:
|
114
|
+
_plugins_actions[plugin_name] = []
|
115
|
+
_plugins_actions[plugin_name].append(func.__name__)
|
116
|
+
if not daemon:
|
117
|
+
_actions_daemon_enabled[func.__name__] = False
|
118
|
+
return func
|
119
|
+
|
120
|
+
if function is None:
|
121
|
+
return _decorator
|
122
|
+
return _decorator(function)
|
91
123
|
|
92
124
|
|
93
125
|
def pre_sync_hook(
|
@@ -115,10 +147,11 @@ def pre_sync_hook(
|
|
115
147
|
>>>
|
116
148
|
"""
|
117
149
|
with _locks['_pre_sync_hooks']:
|
150
|
+
plugin_name = _get_parent_plugin()
|
118
151
|
try:
|
119
|
-
if
|
120
|
-
_pre_sync_hooks[
|
121
|
-
_pre_sync_hooks[
|
152
|
+
if plugin_name not in _pre_sync_hooks:
|
153
|
+
_pre_sync_hooks[plugin_name] = []
|
154
|
+
_pre_sync_hooks[plugin_name].append(function)
|
122
155
|
except Exception as e:
|
123
156
|
from meerschaum.utils.warnings import warn
|
124
157
|
warn(e)
|
@@ -155,9 +188,10 @@ def post_sync_hook(
|
|
155
188
|
"""
|
156
189
|
with _locks['_post_sync_hooks']:
|
157
190
|
try:
|
158
|
-
|
159
|
-
|
160
|
-
|
191
|
+
plugin_name = _get_parent_plugin()
|
192
|
+
if plugin_name not in _post_sync_hooks:
|
193
|
+
_post_sync_hooks[plugin_name] = []
|
194
|
+
_post_sync_hooks[plugin_name].append(function)
|
161
195
|
except Exception as e:
|
162
196
|
from meerschaum.utils.warnings import warn
|
163
197
|
warn(e)
|
@@ -165,6 +199,7 @@ def post_sync_hook(
|
|
165
199
|
|
166
200
|
|
167
201
|
_plugin_endpoints_to_pages = {}
|
202
|
+
_plugins_web_pages = {}
|
168
203
|
def web_page(
|
169
204
|
page: Union[str, None, Callable[[Any], Any]] = None,
|
170
205
|
login_required: bool = True,
|
@@ -199,6 +234,8 @@ def web_page(
|
|
199
234
|
page_str = _func.__name__
|
200
235
|
|
201
236
|
page_str = page_str.lstrip('/').rstrip('/').strip()
|
237
|
+
if not page_str.startswith('dash'):
|
238
|
+
page_str = f'/dash/{page_str}'
|
202
239
|
page_key = (
|
203
240
|
' '.join(
|
204
241
|
[
|
@@ -211,11 +248,7 @@ def web_page(
|
|
211
248
|
)
|
212
249
|
)
|
213
250
|
|
214
|
-
|
215
|
-
plugin_name = (
|
216
|
-
package_name.split('.')[1]
|
217
|
-
if package_name.startswith('plugins.') else None
|
218
|
-
)
|
251
|
+
plugin_name = _get_parent_plugin()
|
219
252
|
page_group = page_group or plugin_name
|
220
253
|
if page_group not in _plugin_endpoints_to_pages:
|
221
254
|
_plugin_endpoints_to_pages[page_group] = {}
|
@@ -225,6 +258,9 @@ def web_page(
|
|
225
258
|
'skip_navbar': skip_navbar,
|
226
259
|
'page_key': page_key,
|
227
260
|
}
|
261
|
+
if plugin_name not in _plugins_web_pages:
|
262
|
+
_plugins_web_pages[plugin_name] = []
|
263
|
+
_plugins_web_pages[plugin_name].append(_func)
|
228
264
|
return wrapper
|
229
265
|
|
230
266
|
if callable(page):
|
@@ -243,10 +279,11 @@ def dash_plugin(function: Callable[[Any], Any]) -> Callable[[Any], Any]:
|
|
243
279
|
Execute the function when starting the Dash application.
|
244
280
|
"""
|
245
281
|
with _locks['_dash_plugins']:
|
282
|
+
plugin_name = _get_parent_plugin()
|
246
283
|
try:
|
247
|
-
if
|
248
|
-
_dash_plugins[
|
249
|
-
_dash_plugins[
|
284
|
+
if plugin_name not in _dash_plugins:
|
285
|
+
_dash_plugins[plugin_name] = []
|
286
|
+
_dash_plugins[plugin_name].append(function)
|
250
287
|
except Exception as e:
|
251
288
|
from meerschaum.utils.warnings import warn
|
252
289
|
warn(e)
|
@@ -284,21 +321,24 @@ def api_plugin(function: Callable[[Any], Any]) -> Callable[[Any], Any]:
|
|
284
321
|
|
285
322
|
|
286
323
|
_synced_symlinks: bool = False
|
324
|
+
_injected_plugin_symlinks = set()
|
287
325
|
def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
|
288
326
|
"""
|
289
|
-
Update the plugins
|
327
|
+
Update the plugins' internal symlinks.
|
290
328
|
"""
|
291
329
|
global _synced_symlinks
|
292
330
|
with _locks['_synced_symlinks']:
|
293
331
|
if _synced_symlinks:
|
294
332
|
return
|
295
333
|
|
296
|
-
import
|
334
|
+
import os
|
335
|
+
import pathlib
|
336
|
+
import time
|
297
337
|
from collections import defaultdict
|
298
338
|
import importlib.util
|
299
339
|
from meerschaum.utils.misc import flatten_list, make_symlink, is_symlink
|
300
340
|
from meerschaum.utils.warnings import error, warn as _warn
|
301
|
-
from meerschaum.
|
341
|
+
from meerschaum._internal.static import STATIC_CONFIG
|
302
342
|
from meerschaum.utils.venv import Venv, activate_venv, deactivate_venv, is_venv_active
|
303
343
|
from meerschaum.config._paths import (
|
304
344
|
PLUGINS_RESOURCES_PATH,
|
@@ -306,6 +346,7 @@ def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
|
|
306
346
|
PLUGINS_INIT_PATH,
|
307
347
|
PLUGINS_DIR_PATHS,
|
308
348
|
PLUGINS_INTERNAL_LOCK_PATH,
|
349
|
+
PLUGINS_INJECTED_RESOURCES_PATH,
|
309
350
|
)
|
310
351
|
|
311
352
|
### If the lock file exists, sleep for up to a second or until it's removed before continuing.
|
@@ -339,7 +380,7 @@ def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
|
|
339
380
|
try:
|
340
381
|
from importlib.metadata import entry_points
|
341
382
|
except ImportError:
|
342
|
-
importlib_metadata = attempt_import('importlib_metadata', lazy=False)
|
383
|
+
importlib_metadata = mrsm.attempt_import('importlib_metadata', lazy=False)
|
343
384
|
entry_points = importlib_metadata.entry_points
|
344
385
|
|
345
386
|
### NOTE: Allow plugins to be installed via `pip`.
|
@@ -367,23 +408,27 @@ def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
|
|
367
408
|
|
368
409
|
PLUGINS_RESOURCES_PATH.mkdir(exist_ok=True)
|
369
410
|
|
370
|
-
existing_symlinked_paths =
|
371
|
-
(
|
411
|
+
existing_symlinked_paths = {
|
412
|
+
_existing_symlink: pathlib.Path(os.path.realpath(_existing_symlink))
|
372
413
|
for item in os.listdir(PLUGINS_RESOURCES_PATH)
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
414
|
+
if is_symlink(_existing_symlink := (PLUGINS_RESOURCES_PATH / item))
|
415
|
+
}
|
416
|
+
injected_symlinked_paths = {
|
417
|
+
_injected_symlink: pathlib.Path(os.path.realpath(_injected_symlink))
|
418
|
+
for item in os.listdir(PLUGINS_INJECTED_RESOURCES_PATH)
|
419
|
+
if is_symlink(_injected_symlink := (PLUGINS_INJECTED_RESOURCES_PATH / item))
|
420
|
+
}
|
377
421
|
plugins_to_be_symlinked = list(flatten_list(
|
378
422
|
[
|
379
423
|
[
|
380
|
-
(plugins_path / item)
|
424
|
+
pathlib.Path(os.path.realpath(plugins_path / item))
|
381
425
|
for item in os.listdir(plugins_path)
|
382
426
|
if (
|
383
427
|
not item.startswith('.')
|
384
428
|
) and (item not in ('__pycache__', '__init__.py'))
|
385
429
|
]
|
386
430
|
for plugins_path in PLUGINS_DIR_PATHS
|
431
|
+
if plugins_path.exists()
|
387
432
|
]
|
388
433
|
))
|
389
434
|
plugins_to_be_symlinked.extend(packaged_plugin_paths)
|
@@ -398,16 +443,19 @@ def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
|
|
398
443
|
if warn:
|
399
444
|
_warn(f"Found duplicate plugins named '{plugin_name}'.")
|
400
445
|
|
401
|
-
for plugin_symlink_path in existing_symlinked_paths:
|
402
|
-
real_path = pathlib.Path(os.path.realpath(plugin_symlink_path))
|
446
|
+
for plugin_symlink_path, real_path in existing_symlinked_paths.items():
|
403
447
|
|
404
448
|
### Remove invalid symlinks.
|
405
449
|
if real_path not in plugins_to_be_symlinked:
|
406
|
-
if
|
450
|
+
if plugin_symlink_path in _injected_plugin_symlinks:
|
451
|
+
continue
|
452
|
+
if plugin_symlink_path in injected_symlinked_paths:
|
453
|
+
continue
|
454
|
+
if real_path in injected_symlinked_paths.values():
|
407
455
|
continue
|
408
456
|
try:
|
409
457
|
plugin_symlink_path.unlink()
|
410
|
-
except Exception
|
458
|
+
except Exception:
|
411
459
|
pass
|
412
460
|
|
413
461
|
### Remove valid plugins from the to-be-symlinked list.
|
@@ -484,13 +532,13 @@ def import_plugins(
|
|
484
532
|
|
485
533
|
"""
|
486
534
|
import sys
|
487
|
-
import os
|
488
535
|
import importlib
|
489
536
|
from meerschaum.utils.misc import flatten_list
|
490
537
|
from meerschaum.config._paths import PLUGINS_RESOURCES_PATH
|
491
538
|
from meerschaum.utils.venv import is_venv_active, activate_venv, deactivate_venv, Venv
|
492
539
|
from meerschaum.utils.warnings import warn as _warn
|
493
540
|
plugins_to_import = list(plugins_to_import)
|
541
|
+
prepended_sys_path = False
|
494
542
|
with _locks['sys.path']:
|
495
543
|
|
496
544
|
### Since plugins may depend on other plugins,
|
@@ -503,6 +551,7 @@ def import_plugins(
|
|
503
551
|
already_active_venvs = [is_venv_active(plugin_name) for plugin_name in plugins_names]
|
504
552
|
|
505
553
|
if not sys.path or sys.path[0] != str(PLUGINS_RESOURCES_PATH.parent):
|
554
|
+
prepended_sys_path = True
|
506
555
|
sys.path.insert(0, str(PLUGINS_RESOURCES_PATH.parent))
|
507
556
|
|
508
557
|
if not plugins_to_import:
|
@@ -548,9 +597,9 @@ def import_plugins(
|
|
548
597
|
imported_plugins.append(None)
|
549
598
|
|
550
599
|
if imported_plugins is None and warn:
|
551
|
-
_warn(
|
600
|
+
_warn("Failed to import plugins.", stacklevel=3)
|
552
601
|
|
553
|
-
if str(PLUGINS_RESOURCES_PATH.parent) in sys.path:
|
602
|
+
if prepended_sys_path and str(PLUGINS_RESOURCES_PATH.parent) in sys.path:
|
554
603
|
sys.path.remove(str(PLUGINS_RESOURCES_PATH.parent))
|
555
604
|
|
556
605
|
if isinstance(imported_plugins, list):
|
@@ -608,7 +657,7 @@ def from_plugin_import(plugin_import_name: str, *attrs: str) -> Any:
|
|
608
657
|
attrs_to_return = []
|
609
658
|
with mrsm.Venv(plugin):
|
610
659
|
if plugin.module is None:
|
611
|
-
|
660
|
+
raise ImportError(f"Unable to import plugin '{plugin}'.")
|
612
661
|
|
613
662
|
try:
|
614
663
|
submodule = importlib.import_module(submodule_import_name)
|
@@ -641,16 +690,23 @@ def from_plugin_import(plugin_import_name: str, *attrs: str) -> Any:
|
|
641
690
|
return tuple(attrs_to_return)
|
642
691
|
|
643
692
|
|
644
|
-
|
693
|
+
_loaded_plugins: bool = False
|
694
|
+
def load_plugins(
|
695
|
+
skip_if_loaded: bool = True,
|
696
|
+
shell: bool = False,
|
697
|
+
debug: bool = False,
|
698
|
+
) -> None:
|
645
699
|
"""
|
646
700
|
Import Meerschaum plugins and update the actions dictionary.
|
647
701
|
"""
|
702
|
+
global _loaded_plugins
|
703
|
+
if skip_if_loaded and _loaded_plugins:
|
704
|
+
return
|
705
|
+
|
648
706
|
from inspect import isfunction, getmembers
|
649
707
|
from meerschaum.actions import __all__ as _all, modules
|
650
708
|
from meerschaum.config._paths import PLUGINS_RESOURCES_PATH
|
651
709
|
from meerschaum.utils.packages import get_modules_from_package
|
652
|
-
if debug:
|
653
|
-
from meerschaum.utils.debug import dprint
|
654
710
|
|
655
711
|
_plugins_names, plugins_modules = get_modules_from_package(
|
656
712
|
import_plugins(),
|
@@ -661,7 +717,8 @@ def load_plugins(debug: bool = False, shell: bool = False) -> None:
|
|
661
717
|
### I'm appending here to keep from redefining the modules list.
|
662
718
|
new_modules = (
|
663
719
|
[
|
664
|
-
mod
|
720
|
+
mod
|
721
|
+
for mod in modules
|
665
722
|
if not mod.__name__.startswith(PLUGINS_RESOURCES_PATH.stem + '.')
|
666
723
|
]
|
667
724
|
+ plugins_modules
|
@@ -677,7 +734,120 @@ def load_plugins(debug: bool = False, shell: bool = False) -> None:
|
|
677
734
|
if not isfunction(func):
|
678
735
|
continue
|
679
736
|
if name == module.__name__.split('.')[-1]:
|
680
|
-
make_action(
|
737
|
+
make_action(
|
738
|
+
func,
|
739
|
+
**{'shell': shell, 'debug': debug},
|
740
|
+
_plugin_name=name,
|
741
|
+
skip_if_loaded=True,
|
742
|
+
)
|
743
|
+
|
744
|
+
_loaded_plugins = True
|
745
|
+
|
746
|
+
|
747
|
+
def unload_custom_actions(plugins: Optional[List[str]] = None, debug: bool = False) -> None:
|
748
|
+
"""
|
749
|
+
Unload the custom actions added by plugins.
|
750
|
+
"""
|
751
|
+
from meerschaum.actions import (
|
752
|
+
actions,
|
753
|
+
_custom_actions_plugins,
|
754
|
+
_plugins_actions,
|
755
|
+
)
|
756
|
+
from meerschaum._internal.entry import _shell
|
757
|
+
import meerschaum._internal.shell as shell_pkg
|
758
|
+
|
759
|
+
plugins = plugins or list(_plugins_actions.keys())
|
760
|
+
|
761
|
+
for plugin in plugins:
|
762
|
+
action_names = _plugins_actions.get(plugin, [])
|
763
|
+
actions_to_remove = {
|
764
|
+
action_name: actions.get(action_name, None)
|
765
|
+
for action_name in action_names
|
766
|
+
}
|
767
|
+
for action_name in action_names:
|
768
|
+
_ = actions.pop(action_name, None)
|
769
|
+
_ = _custom_actions_plugins.pop(action_name, None)
|
770
|
+
_ = _actions_daemon_enabled.pop(action_name, None)
|
771
|
+
|
772
|
+
_ = _plugins_actions.pop(plugin, None)
|
773
|
+
shell_pkg._remove_shell_actions(
|
774
|
+
_shell=_shell,
|
775
|
+
actions=actions_to_remove,
|
776
|
+
)
|
777
|
+
|
778
|
+
|
779
|
+
def unload_plugins(
|
780
|
+
plugins: Optional[List[str]] = None,
|
781
|
+
remove_symlinks: bool = True,
|
782
|
+
debug: bool = False,
|
783
|
+
) -> None:
|
784
|
+
"""
|
785
|
+
Unload the specified plugins from memory.
|
786
|
+
"""
|
787
|
+
global _loaded_plugins
|
788
|
+
import sys
|
789
|
+
from meerschaum.config.paths import PLUGINS_RESOURCES_PATH, PLUGINS_INJECTED_RESOURCES_PATH
|
790
|
+
from meerschaum.connectors import unload_plugin_connectors
|
791
|
+
if debug:
|
792
|
+
from meerschaum.utils.warnings import dprint
|
793
|
+
|
794
|
+
_loaded_plugins = False
|
795
|
+
|
796
|
+
plugins = plugins or get_plugins_names()
|
797
|
+
if debug:
|
798
|
+
dprint(f"Unloading plugins: {plugins}")
|
799
|
+
|
800
|
+
unload_custom_actions(plugins, debug=debug)
|
801
|
+
unload_plugin_connectors(plugins, debug=debug)
|
802
|
+
|
803
|
+
module_prefix = f"{PLUGINS_RESOURCES_PATH.stem}."
|
804
|
+
loaded_modules = [mod_name for mod_name in sys.modules if mod_name.startswith(module_prefix)]
|
805
|
+
|
806
|
+
for plugin_name in plugins:
|
807
|
+
for mod_name in loaded_modules:
|
808
|
+
if mod_name[len(PLUGINS_RESOURCES_PATH.stem):].startswith(plugin_name):
|
809
|
+
_ = sys.modules.pop(mod_name, None)
|
810
|
+
|
811
|
+
### Unload sync hooks.
|
812
|
+
_ = _pre_sync_hooks.pop(plugin_name, None)
|
813
|
+
_ = _post_sync_hooks.pop(plugin_name, None)
|
814
|
+
|
815
|
+
### Unload API endpoints and pages.
|
816
|
+
_ = _dash_plugins.pop(plugin_name, None)
|
817
|
+
web_page_funcs = _plugins_web_pages.pop(plugin_name, None) or []
|
818
|
+
page_groups_to_pop = []
|
819
|
+
for page_group, page_functions in _plugin_endpoints_to_pages.items():
|
820
|
+
page_functions_to_pop = [
|
821
|
+
page_str
|
822
|
+
for page_str, page_payload in page_functions.items()
|
823
|
+
if page_payload.get('function', None) in web_page_funcs
|
824
|
+
]
|
825
|
+
for page_str in page_functions_to_pop:
|
826
|
+
page_functions.pop(page_str, None)
|
827
|
+
if not page_functions:
|
828
|
+
page_groups_to_pop.append(page_group)
|
829
|
+
|
830
|
+
for page_group in page_groups_to_pop:
|
831
|
+
_plugin_endpoints_to_pages.pop(page_group, None)
|
832
|
+
|
833
|
+
### Remove all but injected symlinks.
|
834
|
+
if remove_symlinks:
|
835
|
+
dir_symlink_path = PLUGINS_RESOURCES_PATH / plugin_name
|
836
|
+
dir_symlink_injected_path = PLUGINS_INJECTED_RESOURCES_PATH / plugin_name
|
837
|
+
file_symlink_path = PLUGINS_RESOURCES_PATH / f"{plugin_name}.py"
|
838
|
+
file_symlink_injected_path = PLUGINS_INJECTED_RESOURCES_PATH / f"{plugin_name}.py"
|
839
|
+
|
840
|
+
try:
|
841
|
+
if dir_symlink_path.exists() and not dir_symlink_injected_path.exists():
|
842
|
+
dir_symlink_path.unlink()
|
843
|
+
except Exception:
|
844
|
+
pass
|
845
|
+
|
846
|
+
try:
|
847
|
+
if file_symlink_path.exists() and not file_symlink_injected_path.exists():
|
848
|
+
file_symlink_path.unlink()
|
849
|
+
except Exception:
|
850
|
+
pass
|
681
851
|
|
682
852
|
|
683
853
|
def reload_plugins(plugins: Optional[List[str]] = None, debug: bool = False) -> None:
|
@@ -690,19 +860,11 @@ def reload_plugins(plugins: Optional[List[str]] = None, debug: bool = False) ->
|
|
690
860
|
The plugins to reload. `None` will reload all plugins.
|
691
861
|
|
692
862
|
"""
|
693
|
-
|
694
|
-
|
695
|
-
|
696
|
-
|
697
|
-
|
698
|
-
plugins = get_plugins_names()
|
699
|
-
for plugin_name in plugins:
|
700
|
-
if debug:
|
701
|
-
dprint(f"Reloading plugin '{plugin_name}'...")
|
702
|
-
mod_name = 'plugins.' + str(plugin_name)
|
703
|
-
if mod_name in sys.modules:
|
704
|
-
del sys.modules[mod_name]
|
705
|
-
load_plugins(debug=debug)
|
863
|
+
global _synced_symlinks
|
864
|
+
unload_plugins(plugins, debug=debug)
|
865
|
+
_synced_symlinks = False
|
866
|
+
sync_plugins_symlinks(debug=debug)
|
867
|
+
load_plugins(skip_if_loaded=False, debug=debug)
|
706
868
|
|
707
869
|
|
708
870
|
def get_plugins(*to_load, try_import: bool = True) -> Union[Tuple[Plugin], Plugin]:
|
@@ -722,7 +884,8 @@ def get_plugins(*to_load, try_import: bool = True) -> Union[Tuple[Plugin], Plugi
|
|
722
884
|
import os
|
723
885
|
sync_plugins_symlinks()
|
724
886
|
_plugins = [
|
725
|
-
Plugin(name)
|
887
|
+
Plugin(name)
|
888
|
+
for name in (
|
726
889
|
to_load or [
|
727
890
|
(
|
728
891
|
name if (PLUGINS_RESOURCES_PATH / name).is_dir()
|
@@ -782,8 +945,8 @@ def add_plugin_argument(*args, **kwargs) -> None:
|
|
782
945
|
>>>
|
783
946
|
"""
|
784
947
|
from meerschaum._internal.arguments._parser import groups, _seen_plugin_args, parser
|
785
|
-
from meerschaum.utils.warnings import warn
|
786
|
-
_parent_plugin_name = _get_parent_plugin(
|
948
|
+
from meerschaum.utils.warnings import warn
|
949
|
+
_parent_plugin_name = _get_parent_plugin()
|
787
950
|
title = f"Plugin '{_parent_plugin_name}' options" if _parent_plugin_name else 'Custom options'
|
788
951
|
group_key = 'plugin_' + (_parent_plugin_name or '')
|
789
952
|
if group_key not in groups:
|
@@ -799,12 +962,62 @@ def add_plugin_argument(*args, **kwargs) -> None:
|
|
799
962
|
warn(e)
|
800
963
|
|
801
964
|
|
802
|
-
def
|
965
|
+
def inject_plugin_path(
|
966
|
+
plugin_path: pathlib.Path,
|
967
|
+
plugins_resources_path: Optional[pathlib.Path] = None) -> None:
|
968
|
+
"""
|
969
|
+
Inject a plugin as a symlink into the internal `plugins` directory.
|
970
|
+
|
971
|
+
Parameters
|
972
|
+
----------
|
973
|
+
plugin_path: pathlib.Path
|
974
|
+
The path to the plugin's source module.
|
975
|
+
"""
|
976
|
+
from meerschaum.utils.misc import make_symlink
|
977
|
+
if plugins_resources_path is None:
|
978
|
+
from meerschaum.config.paths import PLUGINS_RESOURCES_PATH, PLUGINS_INJECTED_RESOURCES_PATH
|
979
|
+
plugins_resources_path = PLUGINS_RESOURCES_PATH
|
980
|
+
plugins_injected_resources_path = PLUGINS_INJECTED_RESOURCES_PATH
|
981
|
+
else:
|
982
|
+
plugins_injected_resources_path = plugins_resources_path / '.injected'
|
983
|
+
|
984
|
+
if plugin_path.is_dir():
|
985
|
+
plugin_name = plugin_path.name
|
986
|
+
dest_path = plugins_resources_path / plugin_name
|
987
|
+
injected_path = plugins_injected_resources_path / plugin_name
|
988
|
+
elif plugin_path.name == '__init__.py':
|
989
|
+
plugin_name = plugin_path.parent.name
|
990
|
+
dest_path = plugins_resources_path / plugin_name
|
991
|
+
injected_path = plugins_injected_resources_path / plugin_name
|
992
|
+
elif plugin_path.name.endswith('.py'):
|
993
|
+
plugin_name = plugin_path.name[:(-1 * len('.py'))]
|
994
|
+
dest_path = plugins_resources_path / plugin_path.name
|
995
|
+
injected_path = plugins_injected_resources_path / plugin_path.name
|
996
|
+
else:
|
997
|
+
raise ValueError(f"Cannot deduce plugin name from path '{plugin_path}'.")
|
998
|
+
|
999
|
+
_injected_plugin_symlinks.add(dest_path)
|
1000
|
+
make_symlink(plugin_path, dest_path)
|
1001
|
+
make_symlink(plugin_path, injected_path)
|
1002
|
+
|
1003
|
+
|
1004
|
+
def _get_parent_plugin(stacklevel: Union[int, Tuple[int, ...]] = (1, 2, 3, 4)) -> Union[str, None]:
|
803
1005
|
"""If this function is called from outside a Meerschaum plugin, it will return None."""
|
804
|
-
import inspect
|
805
|
-
|
806
|
-
|
807
|
-
|
808
|
-
|
809
|
-
|
810
|
-
|
1006
|
+
import inspect
|
1007
|
+
if not isinstance(stacklevel, tuple):
|
1008
|
+
stacklevel = (stacklevel,)
|
1009
|
+
|
1010
|
+
for _level in stacklevel:
|
1011
|
+
try:
|
1012
|
+
parent_globals = inspect.stack()[_level][0].f_globals
|
1013
|
+
global_name = parent_globals.get('__name__', '')
|
1014
|
+
if global_name.startswith('meerschaum.'):
|
1015
|
+
continue
|
1016
|
+
plugin_name = global_name.replace('plugins.', '').split('.')[0]
|
1017
|
+
if plugin_name.startswith('_') or plugin_name == 'importlib':
|
1018
|
+
continue
|
1019
|
+
return plugin_name
|
1020
|
+
except Exception:
|
1021
|
+
continue
|
1022
|
+
|
1023
|
+
return None
|