meerschaum 2.9.4__py3-none-any.whl → 3.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (201) hide show
  1. meerschaum/__init__.py +5 -2
  2. meerschaum/_internal/__init__.py +1 -0
  3. meerschaum/_internal/arguments/_parse_arguments.py +4 -4
  4. meerschaum/_internal/arguments/_parser.py +33 -4
  5. meerschaum/_internal/cli/__init__.py +6 -0
  6. meerschaum/_internal/cli/daemons.py +103 -0
  7. meerschaum/_internal/cli/entry.py +220 -0
  8. meerschaum/_internal/cli/workers.py +435 -0
  9. meerschaum/_internal/docs/index.py +48 -2
  10. meerschaum/_internal/entry.py +50 -14
  11. meerschaum/_internal/shell/Shell.py +121 -29
  12. meerschaum/_internal/shell/__init__.py +4 -1
  13. meerschaum/_internal/static.py +359 -0
  14. meerschaum/_internal/term/TermPageHandler.py +1 -2
  15. meerschaum/_internal/term/__init__.py +40 -6
  16. meerschaum/_internal/term/tools.py +33 -8
  17. meerschaum/actions/__init__.py +6 -4
  18. meerschaum/actions/api.py +53 -13
  19. meerschaum/actions/attach.py +1 -0
  20. meerschaum/actions/bootstrap.py +8 -8
  21. meerschaum/actions/delete.py +4 -2
  22. meerschaum/actions/edit.py +171 -25
  23. meerschaum/actions/login.py +8 -8
  24. meerschaum/actions/register.py +143 -6
  25. meerschaum/actions/reload.py +22 -5
  26. meerschaum/actions/restart.py +14 -0
  27. meerschaum/actions/show.py +184 -31
  28. meerschaum/actions/start.py +166 -17
  29. meerschaum/actions/stop.py +38 -2
  30. meerschaum/actions/sync.py +7 -2
  31. meerschaum/actions/tag.py +9 -8
  32. meerschaum/actions/verify.py +5 -8
  33. meerschaum/api/__init__.py +45 -15
  34. meerschaum/api/_events.py +46 -4
  35. meerschaum/api/_oauth2.py +162 -9
  36. meerschaum/api/_tokens.py +102 -0
  37. meerschaum/api/dash/__init__.py +0 -3
  38. meerschaum/api/dash/callbacks/__init__.py +1 -0
  39. meerschaum/api/dash/callbacks/custom.py +4 -3
  40. meerschaum/api/dash/callbacks/dashboard.py +228 -117
  41. meerschaum/api/dash/callbacks/jobs.py +14 -7
  42. meerschaum/api/dash/callbacks/login.py +10 -1
  43. meerschaum/api/dash/callbacks/pipes.py +194 -14
  44. meerschaum/api/dash/callbacks/plugins.py +0 -1
  45. meerschaum/api/dash/callbacks/register.py +10 -3
  46. meerschaum/api/dash/callbacks/settings/password_reset.py +2 -2
  47. meerschaum/api/dash/callbacks/tokens.py +389 -0
  48. meerschaum/api/dash/components.py +36 -15
  49. meerschaum/api/dash/jobs.py +1 -1
  50. meerschaum/api/dash/keys.py +35 -93
  51. meerschaum/api/dash/pages/__init__.py +2 -1
  52. meerschaum/api/dash/pages/dashboard.py +1 -20
  53. meerschaum/api/dash/pages/{job.py → jobs.py} +10 -7
  54. meerschaum/api/dash/pages/login.py +2 -2
  55. meerschaum/api/dash/pages/pipes.py +16 -5
  56. meerschaum/api/dash/pages/settings/password_reset.py +1 -1
  57. meerschaum/api/dash/pages/tokens.py +53 -0
  58. meerschaum/api/dash/pipes.py +438 -88
  59. meerschaum/api/dash/sessions.py +12 -0
  60. meerschaum/api/dash/tokens.py +603 -0
  61. meerschaum/api/dash/websockets.py +1 -1
  62. meerschaum/api/dash/webterm.py +18 -6
  63. meerschaum/api/models/__init__.py +23 -3
  64. meerschaum/api/models/_actions.py +22 -0
  65. meerschaum/api/models/_pipes.py +91 -7
  66. meerschaum/api/models/_tokens.py +81 -0
  67. meerschaum/api/resources/static/css/dash.css +16 -0
  68. meerschaum/api/resources/static/js/terminado.js +3 -0
  69. meerschaum/api/resources/static/js/xterm-addon-unicode11.js +2 -0
  70. meerschaum/api/resources/templates/termpage.html +13 -0
  71. meerschaum/api/routes/__init__.py +1 -0
  72. meerschaum/api/routes/_actions.py +3 -4
  73. meerschaum/api/routes/_connectors.py +3 -7
  74. meerschaum/api/routes/_jobs.py +26 -35
  75. meerschaum/api/routes/_login.py +120 -15
  76. meerschaum/api/routes/_misc.py +5 -10
  77. meerschaum/api/routes/_pipes.py +178 -143
  78. meerschaum/api/routes/_plugins.py +38 -28
  79. meerschaum/api/routes/_tokens.py +236 -0
  80. meerschaum/api/routes/_users.py +47 -35
  81. meerschaum/api/routes/_version.py +3 -3
  82. meerschaum/api/routes/_webterm.py +3 -3
  83. meerschaum/config/__init__.py +100 -30
  84. meerschaum/config/_default.py +132 -64
  85. meerschaum/config/_edit.py +38 -32
  86. meerschaum/config/_formatting.py +2 -0
  87. meerschaum/config/_patch.py +10 -8
  88. meerschaum/config/_paths.py +133 -13
  89. meerschaum/config/_read_config.py +87 -36
  90. meerschaum/config/_sync.py +6 -3
  91. meerschaum/config/_version.py +1 -1
  92. meerschaum/config/environment.py +262 -0
  93. meerschaum/config/stack/__init__.py +37 -15
  94. meerschaum/config/static.py +18 -0
  95. meerschaum/connectors/_Connector.py +11 -6
  96. meerschaum/connectors/__init__.py +41 -22
  97. meerschaum/connectors/api/_APIConnector.py +34 -6
  98. meerschaum/connectors/api/_actions.py +2 -2
  99. meerschaum/connectors/api/_jobs.py +12 -1
  100. meerschaum/connectors/api/_login.py +33 -7
  101. meerschaum/connectors/api/_misc.py +2 -2
  102. meerschaum/connectors/api/_pipes.py +23 -32
  103. meerschaum/connectors/api/_plugins.py +2 -2
  104. meerschaum/connectors/api/_request.py +1 -1
  105. meerschaum/connectors/api/_tokens.py +146 -0
  106. meerschaum/connectors/api/_users.py +70 -58
  107. meerschaum/connectors/instance/_InstanceConnector.py +83 -0
  108. meerschaum/connectors/instance/__init__.py +10 -0
  109. meerschaum/connectors/instance/_pipes.py +442 -0
  110. meerschaum/connectors/instance/_plugins.py +159 -0
  111. meerschaum/connectors/instance/_tokens.py +317 -0
  112. meerschaum/connectors/instance/_users.py +188 -0
  113. meerschaum/connectors/parse.py +5 -2
  114. meerschaum/connectors/sql/_SQLConnector.py +22 -5
  115. meerschaum/connectors/sql/_cli.py +12 -11
  116. meerschaum/connectors/sql/_create_engine.py +12 -168
  117. meerschaum/connectors/sql/_fetch.py +2 -18
  118. meerschaum/connectors/sql/_pipes.py +295 -278
  119. meerschaum/connectors/sql/_plugins.py +29 -0
  120. meerschaum/connectors/sql/_sql.py +47 -22
  121. meerschaum/connectors/sql/_users.py +36 -2
  122. meerschaum/connectors/sql/tables/__init__.py +254 -122
  123. meerschaum/connectors/valkey/_ValkeyConnector.py +5 -7
  124. meerschaum/connectors/valkey/_pipes.py +60 -31
  125. meerschaum/connectors/valkey/_plugins.py +2 -26
  126. meerschaum/core/Pipe/__init__.py +115 -85
  127. meerschaum/core/Pipe/_attributes.py +425 -124
  128. meerschaum/core/Pipe/_bootstrap.py +54 -24
  129. meerschaum/core/Pipe/_cache.py +555 -0
  130. meerschaum/core/Pipe/_clear.py +0 -11
  131. meerschaum/core/Pipe/_data.py +96 -68
  132. meerschaum/core/Pipe/_deduplicate.py +0 -13
  133. meerschaum/core/Pipe/_delete.py +12 -21
  134. meerschaum/core/Pipe/_drop.py +11 -23
  135. meerschaum/core/Pipe/_dtypes.py +49 -19
  136. meerschaum/core/Pipe/_edit.py +14 -4
  137. meerschaum/core/Pipe/_fetch.py +1 -1
  138. meerschaum/core/Pipe/_index.py +8 -14
  139. meerschaum/core/Pipe/_show.py +5 -5
  140. meerschaum/core/Pipe/_sync.py +123 -204
  141. meerschaum/core/Pipe/_verify.py +4 -4
  142. meerschaum/{plugins → core/Plugin}/_Plugin.py +16 -12
  143. meerschaum/core/Plugin/__init__.py +1 -1
  144. meerschaum/core/Token/_Token.py +220 -0
  145. meerschaum/core/Token/__init__.py +12 -0
  146. meerschaum/core/User/_User.py +35 -10
  147. meerschaum/core/User/__init__.py +9 -1
  148. meerschaum/core/__init__.py +1 -0
  149. meerschaum/jobs/_Executor.py +88 -4
  150. meerschaum/jobs/_Job.py +149 -38
  151. meerschaum/jobs/__init__.py +3 -2
  152. meerschaum/jobs/systemd.py +8 -3
  153. meerschaum/models/__init__.py +35 -0
  154. meerschaum/models/pipes.py +247 -0
  155. meerschaum/models/tokens.py +38 -0
  156. meerschaum/models/users.py +26 -0
  157. meerschaum/plugins/__init__.py +301 -88
  158. meerschaum/plugins/bootstrap.py +510 -4
  159. meerschaum/utils/_get_pipes.py +97 -30
  160. meerschaum/utils/daemon/Daemon.py +199 -43
  161. meerschaum/utils/daemon/FileDescriptorInterceptor.py +0 -1
  162. meerschaum/utils/daemon/RotatingFile.py +63 -36
  163. meerschaum/utils/daemon/StdinFile.py +53 -13
  164. meerschaum/utils/daemon/__init__.py +47 -6
  165. meerschaum/utils/daemon/_names.py +6 -3
  166. meerschaum/utils/dataframe.py +480 -82
  167. meerschaum/utils/debug.py +49 -19
  168. meerschaum/utils/dtypes/__init__.py +478 -37
  169. meerschaum/utils/dtypes/sql.py +369 -29
  170. meerschaum/utils/formatting/__init__.py +5 -2
  171. meerschaum/utils/formatting/_jobs.py +1 -1
  172. meerschaum/utils/formatting/_pipes.py +52 -50
  173. meerschaum/utils/formatting/_pprint.py +1 -0
  174. meerschaum/utils/formatting/_shell.py +44 -18
  175. meerschaum/utils/misc.py +268 -186
  176. meerschaum/utils/packages/__init__.py +25 -40
  177. meerschaum/utils/packages/_packages.py +42 -34
  178. meerschaum/utils/pipes.py +213 -0
  179. meerschaum/utils/process.py +2 -2
  180. meerschaum/utils/prompt.py +175 -144
  181. meerschaum/utils/schedule.py +2 -1
  182. meerschaum/utils/sql.py +135 -49
  183. meerschaum/utils/threading.py +42 -0
  184. meerschaum/utils/typing.py +1 -4
  185. meerschaum/utils/venv/_Venv.py +2 -2
  186. meerschaum/utils/venv/__init__.py +7 -7
  187. meerschaum/utils/warnings.py +19 -13
  188. {meerschaum-2.9.4.dist-info → meerschaum-3.0.0.dist-info}/METADATA +94 -96
  189. meerschaum-3.0.0.dist-info/RECORD +289 -0
  190. {meerschaum-2.9.4.dist-info → meerschaum-3.0.0.dist-info}/WHEEL +1 -1
  191. meerschaum-3.0.0.dist-info/licenses/NOTICE +2 -0
  192. meerschaum/api/models/_interfaces.py +0 -15
  193. meerschaum/api/models/_locations.py +0 -15
  194. meerschaum/api/models/_metrics.py +0 -15
  195. meerschaum/config/_environment.py +0 -145
  196. meerschaum/config/static/__init__.py +0 -186
  197. meerschaum-2.9.4.dist-info/RECORD +0 -263
  198. {meerschaum-2.9.4.dist-info → meerschaum-3.0.0.dist-info}/entry_points.txt +0 -0
  199. {meerschaum-2.9.4.dist-info → meerschaum-3.0.0.dist-info}/licenses/LICENSE +0 -0
  200. {meerschaum-2.9.4.dist-info → meerschaum-3.0.0.dist-info}/top_level.txt +0 -0
  201. {meerschaum-2.9.4.dist-info → meerschaum-3.0.0.dist-info}/zip-safe +0 -0
@@ -7,20 +7,25 @@ Expose plugin management APIs from the `meerschaum.plugins` module.
7
7
  """
8
8
 
9
9
  from __future__ import annotations
10
+
11
+ import pathlib
10
12
  import functools
13
+
11
14
  import meerschaum as mrsm
12
15
  from meerschaum.utils.typing import Callable, Any, Union, Optional, Dict, List, Tuple
13
16
  from meerschaum.utils.threading import Lock, RLock
14
- from meerschaum.plugins._Plugin import Plugin
17
+ from meerschaum.core.Plugin import Plugin
15
18
 
16
19
  _api_plugins: Dict[str, List[Callable[['fastapi.App'], Any]]] = {}
17
- _pre_sync_hooks: Dict[str, List[Callable[[Any], Any]]] = {}
18
- _post_sync_hooks: Dict[str, List[Callable[[Any], Any]]] = {}
20
+ _pre_sync_hooks: Dict[Union[str, None], List[Callable[[Any], Any]]] = {}
21
+ _post_sync_hooks: Dict[Union[str, None], List[Callable[[Any], Any]]] = {}
22
+ _actions_daemon_enabled: Dict[str, bool] = {}
19
23
  _locks = {
20
24
  '_api_plugins': RLock(),
21
25
  '_dash_plugins': RLock(),
22
26
  '_pre_sync_hooks': RLock(),
23
27
  '_post_sync_hooks': RLock(),
28
+ '_actions_daemon_enabled': RLock(),
24
29
  '__path__': RLock(),
25
30
  'sys.path': RLock(),
26
31
  'internal_plugins': RLock(),
@@ -28,21 +33,37 @@ _locks = {
28
33
  'PLUGINS_INTERNAL_LOCK_PATH': RLock(),
29
34
  }
30
35
  __all__ = (
31
- "Plugin", "make_action", "api_plugin", "dash_plugin", "web_page",
32
- "import_plugins", "from_plugin_import",
33
- "reload_plugins", "get_plugins", "get_data_plugins", "add_plugin_argument",
34
- "pre_sync_hook", "post_sync_hook",
36
+ "Plugin",
37
+ "make_action",
38
+ "api_plugin",
39
+ "dash_plugin",
40
+ "web_page",
41
+ "import_plugins",
42
+ "from_plugin_import",
43
+ "reload_plugins",
44
+ "get_plugins",
45
+ "get_data_plugins",
46
+ "add_plugin_argument",
47
+ "pre_sync_hook",
48
+ "post_sync_hook",
35
49
  )
36
50
  __pdoc__ = {
37
- 'venvs': False, 'data': False, 'stack': False, 'plugins': False,
51
+ 'venvs': False,
52
+ 'data': False,
53
+ 'stack': False,
54
+ 'plugins': False,
38
55
  }
39
56
 
57
+
40
58
  def make_action(
41
- function: Callable[[Any], Any],
59
+ function: Optional[Callable[[Any], Any]] = None,
42
60
  shell: bool = False,
43
61
  activate: bool = True,
44
62
  deactivate: bool = True,
45
- debug: bool = False
63
+ debug: bool = False,
64
+ daemon: bool = True,
65
+ skip_if_loaded: bool = True,
66
+ _plugin_name: Optional[str] = None,
46
67
  ) -> Callable[[Any], Any]:
47
68
  """
48
69
  Make a function a Meerschaum action. Useful for plugins that are adding multiple actions.
@@ -69,25 +90,36 @@ def make_action(
69
90
  ... return True, "Success"
70
91
  >>>
71
92
  """
93
+ def _decorator(func: Callable[[Any], Any]) -> Callable[[Any], Any]:
94
+ from meerschaum.actions import actions, _custom_actions_plugins, _plugins_actions
95
+ if skip_if_loaded and func.__name__ in actions:
96
+ return func
72
97
 
73
- from meerschaum.actions import actions
74
- from meerschaum.utils.formatting import pprint
75
- package_name = function.__globals__['__name__']
76
- plugin_name = (
77
- package_name.split('.')[1]
78
- if package_name.startswith('plugins.') else None
79
- )
80
- plugin = Plugin(plugin_name) if plugin_name else None
81
-
82
- if debug:
83
- from meerschaum.utils.debug import dprint
84
- dprint(
85
- f"Adding action '{function.__name__}' from plugin " +
86
- f"'{plugin}'..."
98
+ from meerschaum.config.paths import PLUGINS_RESOURCES_PATH
99
+ plugin_name = (
100
+ func.__name__.split(f"{PLUGINS_RESOURCES_PATH.stem}.", maxsplit=1)[-1].split('.')[0]
87
101
  )
102
+ plugin = Plugin(plugin_name) if plugin_name else None
88
103
 
89
- actions[function.__name__] = function
90
- return function
104
+ if debug:
105
+ from meerschaum.utils.debug import dprint
106
+ dprint(
107
+ f"Adding action '{func.__name__}' from plugin "
108
+ f"'{plugin}'..."
109
+ )
110
+
111
+ actions[func.__name__] = func
112
+ _custom_actions_plugins[func.__name__] = plugin_name
113
+ if plugin_name not in _plugins_actions:
114
+ _plugins_actions[plugin_name] = []
115
+ _plugins_actions[plugin_name].append(func.__name__)
116
+ if not daemon:
117
+ _actions_daemon_enabled[func.__name__] = False
118
+ return func
119
+
120
+ if function is None:
121
+ return _decorator
122
+ return _decorator(function)
91
123
 
92
124
 
93
125
  def pre_sync_hook(
@@ -115,10 +147,11 @@ def pre_sync_hook(
115
147
  >>>
116
148
  """
117
149
  with _locks['_pre_sync_hooks']:
150
+ plugin_name = _get_parent_plugin()
118
151
  try:
119
- if function.__module__ not in _pre_sync_hooks:
120
- _pre_sync_hooks[function.__module__] = []
121
- _pre_sync_hooks[function.__module__].append(function)
152
+ if plugin_name not in _pre_sync_hooks:
153
+ _pre_sync_hooks[plugin_name] = []
154
+ _pre_sync_hooks[plugin_name].append(function)
122
155
  except Exception as e:
123
156
  from meerschaum.utils.warnings import warn
124
157
  warn(e)
@@ -155,9 +188,10 @@ def post_sync_hook(
155
188
  """
156
189
  with _locks['_post_sync_hooks']:
157
190
  try:
158
- if function.__module__ not in _post_sync_hooks:
159
- _post_sync_hooks[function.__module__] = []
160
- _post_sync_hooks[function.__module__].append(function)
191
+ plugin_name = _get_parent_plugin()
192
+ if plugin_name not in _post_sync_hooks:
193
+ _post_sync_hooks[plugin_name] = []
194
+ _post_sync_hooks[plugin_name].append(function)
161
195
  except Exception as e:
162
196
  from meerschaum.utils.warnings import warn
163
197
  warn(e)
@@ -165,6 +199,7 @@ def post_sync_hook(
165
199
 
166
200
 
167
201
  _plugin_endpoints_to_pages = {}
202
+ _plugins_web_pages = {}
168
203
  def web_page(
169
204
  page: Union[str, None, Callable[[Any], Any]] = None,
170
205
  login_required: bool = True,
@@ -199,6 +234,8 @@ def web_page(
199
234
  page_str = _func.__name__
200
235
 
201
236
  page_str = page_str.lstrip('/').rstrip('/').strip()
237
+ if not page_str.startswith('dash'):
238
+ page_str = f'/dash/{page_str}'
202
239
  page_key = (
203
240
  ' '.join(
204
241
  [
@@ -211,11 +248,7 @@ def web_page(
211
248
  )
212
249
  )
213
250
 
214
- package_name = _func.__globals__['__name__']
215
- plugin_name = (
216
- package_name.split('.')[1]
217
- if package_name.startswith('plugins.') else None
218
- )
251
+ plugin_name = _get_parent_plugin()
219
252
  page_group = page_group or plugin_name
220
253
  if page_group not in _plugin_endpoints_to_pages:
221
254
  _plugin_endpoints_to_pages[page_group] = {}
@@ -225,6 +258,9 @@ def web_page(
225
258
  'skip_navbar': skip_navbar,
226
259
  'page_key': page_key,
227
260
  }
261
+ if plugin_name not in _plugins_web_pages:
262
+ _plugins_web_pages[plugin_name] = []
263
+ _plugins_web_pages[plugin_name].append(_func)
228
264
  return wrapper
229
265
 
230
266
  if callable(page):
@@ -243,10 +279,11 @@ def dash_plugin(function: Callable[[Any], Any]) -> Callable[[Any], Any]:
243
279
  Execute the function when starting the Dash application.
244
280
  """
245
281
  with _locks['_dash_plugins']:
282
+ plugin_name = _get_parent_plugin()
246
283
  try:
247
- if function.__module__ not in _dash_plugins:
248
- _dash_plugins[function.__module__] = []
249
- _dash_plugins[function.__module__].append(function)
284
+ if plugin_name not in _dash_plugins:
285
+ _dash_plugins[plugin_name] = []
286
+ _dash_plugins[plugin_name].append(function)
250
287
  except Exception as e:
251
288
  from meerschaum.utils.warnings import warn
252
289
  warn(e)
@@ -284,21 +321,24 @@ def api_plugin(function: Callable[[Any], Any]) -> Callable[[Any], Any]:
284
321
 
285
322
 
286
323
  _synced_symlinks: bool = False
324
+ _injected_plugin_symlinks = set()
287
325
  def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
288
326
  """
289
- Update the plugins
327
+ Update the plugins' internal symlinks.
290
328
  """
291
329
  global _synced_symlinks
292
330
  with _locks['_synced_symlinks']:
293
331
  if _synced_symlinks:
294
332
  return
295
333
 
296
- import sys, os, pathlib, time
334
+ import os
335
+ import pathlib
336
+ import time
297
337
  from collections import defaultdict
298
338
  import importlib.util
299
339
  from meerschaum.utils.misc import flatten_list, make_symlink, is_symlink
300
340
  from meerschaum.utils.warnings import error, warn as _warn
301
- from meerschaum.config.static import STATIC_CONFIG
341
+ from meerschaum._internal.static import STATIC_CONFIG
302
342
  from meerschaum.utils.venv import Venv, activate_venv, deactivate_venv, is_venv_active
303
343
  from meerschaum.config._paths import (
304
344
  PLUGINS_RESOURCES_PATH,
@@ -306,6 +346,7 @@ def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
306
346
  PLUGINS_INIT_PATH,
307
347
  PLUGINS_DIR_PATHS,
308
348
  PLUGINS_INTERNAL_LOCK_PATH,
349
+ PLUGINS_INJECTED_RESOURCES_PATH,
309
350
  )
310
351
 
311
352
  ### If the lock file exists, sleep for up to a second or until it's removed before continuing.
@@ -339,7 +380,7 @@ def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
339
380
  try:
340
381
  from importlib.metadata import entry_points
341
382
  except ImportError:
342
- importlib_metadata = attempt_import('importlib_metadata', lazy=False)
383
+ importlib_metadata = mrsm.attempt_import('importlib_metadata', lazy=False)
343
384
  entry_points = importlib_metadata.entry_points
344
385
 
345
386
  ### NOTE: Allow plugins to be installed via `pip`.
@@ -367,23 +408,27 @@ def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
367
408
 
368
409
  PLUGINS_RESOURCES_PATH.mkdir(exist_ok=True)
369
410
 
370
- existing_symlinked_paths = [
371
- (PLUGINS_RESOURCES_PATH / item)
411
+ existing_symlinked_paths = {
412
+ _existing_symlink: pathlib.Path(os.path.realpath(_existing_symlink))
372
413
  for item in os.listdir(PLUGINS_RESOURCES_PATH)
373
- ]
374
- for plugins_path in PLUGINS_DIR_PATHS:
375
- if not plugins_path.exists():
376
- plugins_path.mkdir(exist_ok=True, parents=True)
414
+ if is_symlink(_existing_symlink := (PLUGINS_RESOURCES_PATH / item))
415
+ }
416
+ injected_symlinked_paths = {
417
+ _injected_symlink: pathlib.Path(os.path.realpath(_injected_symlink))
418
+ for item in os.listdir(PLUGINS_INJECTED_RESOURCES_PATH)
419
+ if is_symlink(_injected_symlink := (PLUGINS_INJECTED_RESOURCES_PATH / item))
420
+ }
377
421
  plugins_to_be_symlinked = list(flatten_list(
378
422
  [
379
423
  [
380
- (plugins_path / item)
424
+ pathlib.Path(os.path.realpath(plugins_path / item))
381
425
  for item in os.listdir(plugins_path)
382
426
  if (
383
427
  not item.startswith('.')
384
428
  ) and (item not in ('__pycache__', '__init__.py'))
385
429
  ]
386
430
  for plugins_path in PLUGINS_DIR_PATHS
431
+ if plugins_path.exists()
387
432
  ]
388
433
  ))
389
434
  plugins_to_be_symlinked.extend(packaged_plugin_paths)
@@ -398,16 +443,19 @@ def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
398
443
  if warn:
399
444
  _warn(f"Found duplicate plugins named '{plugin_name}'.")
400
445
 
401
- for plugin_symlink_path in existing_symlinked_paths:
402
- real_path = pathlib.Path(os.path.realpath(plugin_symlink_path))
446
+ for plugin_symlink_path, real_path in existing_symlinked_paths.items():
403
447
 
404
448
  ### Remove invalid symlinks.
405
449
  if real_path not in plugins_to_be_symlinked:
406
- if not is_symlink(plugin_symlink_path):
450
+ if plugin_symlink_path in _injected_plugin_symlinks:
451
+ continue
452
+ if plugin_symlink_path in injected_symlinked_paths:
453
+ continue
454
+ if real_path in injected_symlinked_paths.values():
407
455
  continue
408
456
  try:
409
457
  plugin_symlink_path.unlink()
410
- except Exception as e:
458
+ except Exception:
411
459
  pass
412
460
 
413
461
  ### Remove valid plugins from the to-be-symlinked list.
@@ -484,13 +532,13 @@ def import_plugins(
484
532
 
485
533
  """
486
534
  import sys
487
- import os
488
535
  import importlib
489
536
  from meerschaum.utils.misc import flatten_list
490
537
  from meerschaum.config._paths import PLUGINS_RESOURCES_PATH
491
538
  from meerschaum.utils.venv import is_venv_active, activate_venv, deactivate_venv, Venv
492
539
  from meerschaum.utils.warnings import warn as _warn
493
540
  plugins_to_import = list(plugins_to_import)
541
+ prepended_sys_path = False
494
542
  with _locks['sys.path']:
495
543
 
496
544
  ### Since plugins may depend on other plugins,
@@ -503,6 +551,7 @@ def import_plugins(
503
551
  already_active_venvs = [is_venv_active(plugin_name) for plugin_name in plugins_names]
504
552
 
505
553
  if not sys.path or sys.path[0] != str(PLUGINS_RESOURCES_PATH.parent):
554
+ prepended_sys_path = True
506
555
  sys.path.insert(0, str(PLUGINS_RESOURCES_PATH.parent))
507
556
 
508
557
  if not plugins_to_import:
@@ -548,9 +597,9 @@ def import_plugins(
548
597
  imported_plugins.append(None)
549
598
 
550
599
  if imported_plugins is None and warn:
551
- _warn(f"Failed to import plugins.", stacklevel=3)
600
+ _warn("Failed to import plugins.", stacklevel=3)
552
601
 
553
- if str(PLUGINS_RESOURCES_PATH.parent) in sys.path:
602
+ if prepended_sys_path and str(PLUGINS_RESOURCES_PATH.parent) in sys.path:
554
603
  sys.path.remove(str(PLUGINS_RESOURCES_PATH.parent))
555
604
 
556
605
  if isinstance(imported_plugins, list):
@@ -608,7 +657,7 @@ def from_plugin_import(plugin_import_name: str, *attrs: str) -> Any:
608
657
  attrs_to_return = []
609
658
  with mrsm.Venv(plugin):
610
659
  if plugin.module is None:
611
- return None
660
+ raise ImportError(f"Unable to import plugin '{plugin}'.")
612
661
 
613
662
  try:
614
663
  submodule = importlib.import_module(submodule_import_name)
@@ -641,16 +690,23 @@ def from_plugin_import(plugin_import_name: str, *attrs: str) -> Any:
641
690
  return tuple(attrs_to_return)
642
691
 
643
692
 
644
- def load_plugins(debug: bool = False, shell: bool = False) -> None:
693
+ _loaded_plugins: bool = False
694
+ def load_plugins(
695
+ skip_if_loaded: bool = True,
696
+ shell: bool = False,
697
+ debug: bool = False,
698
+ ) -> None:
645
699
  """
646
700
  Import Meerschaum plugins and update the actions dictionary.
647
701
  """
702
+ global _loaded_plugins
703
+ if skip_if_loaded and _loaded_plugins:
704
+ return
705
+
648
706
  from inspect import isfunction, getmembers
649
707
  from meerschaum.actions import __all__ as _all, modules
650
708
  from meerschaum.config._paths import PLUGINS_RESOURCES_PATH
651
709
  from meerschaum.utils.packages import get_modules_from_package
652
- if debug:
653
- from meerschaum.utils.debug import dprint
654
710
 
655
711
  _plugins_names, plugins_modules = get_modules_from_package(
656
712
  import_plugins(),
@@ -661,7 +717,8 @@ def load_plugins(debug: bool = False, shell: bool = False) -> None:
661
717
  ### I'm appending here to keep from redefining the modules list.
662
718
  new_modules = (
663
719
  [
664
- mod for mod in modules
720
+ mod
721
+ for mod in modules
665
722
  if not mod.__name__.startswith(PLUGINS_RESOURCES_PATH.stem + '.')
666
723
  ]
667
724
  + plugins_modules
@@ -677,7 +734,120 @@ def load_plugins(debug: bool = False, shell: bool = False) -> None:
677
734
  if not isfunction(func):
678
735
  continue
679
736
  if name == module.__name__.split('.')[-1]:
680
- make_action(func, **{'shell': shell, 'debug': debug})
737
+ make_action(
738
+ func,
739
+ **{'shell': shell, 'debug': debug},
740
+ _plugin_name=name,
741
+ skip_if_loaded=True,
742
+ )
743
+
744
+ _loaded_plugins = True
745
+
746
+
747
+ def unload_custom_actions(plugins: Optional[List[str]] = None, debug: bool = False) -> None:
748
+ """
749
+ Unload the custom actions added by plugins.
750
+ """
751
+ from meerschaum.actions import (
752
+ actions,
753
+ _custom_actions_plugins,
754
+ _plugins_actions,
755
+ )
756
+ from meerschaum._internal.entry import _shell
757
+ import meerschaum._internal.shell as shell_pkg
758
+
759
+ plugins = plugins or list(_plugins_actions.keys())
760
+
761
+ for plugin in plugins:
762
+ action_names = _plugins_actions.get(plugin, [])
763
+ actions_to_remove = {
764
+ action_name: actions.get(action_name, None)
765
+ for action_name in action_names
766
+ }
767
+ for action_name in action_names:
768
+ _ = actions.pop(action_name, None)
769
+ _ = _custom_actions_plugins.pop(action_name, None)
770
+ _ = _actions_daemon_enabled.pop(action_name, None)
771
+
772
+ _ = _plugins_actions.pop(plugin, None)
773
+ shell_pkg._remove_shell_actions(
774
+ _shell=_shell,
775
+ actions=actions_to_remove,
776
+ )
777
+
778
+
779
+ def unload_plugins(
780
+ plugins: Optional[List[str]] = None,
781
+ remove_symlinks: bool = True,
782
+ debug: bool = False,
783
+ ) -> None:
784
+ """
785
+ Unload the specified plugins from memory.
786
+ """
787
+ global _loaded_plugins
788
+ import sys
789
+ from meerschaum.config.paths import PLUGINS_RESOURCES_PATH, PLUGINS_INJECTED_RESOURCES_PATH
790
+ from meerschaum.connectors import unload_plugin_connectors
791
+ if debug:
792
+ from meerschaum.utils.warnings import dprint
793
+
794
+ _loaded_plugins = False
795
+
796
+ plugins = plugins or get_plugins_names()
797
+ if debug:
798
+ dprint(f"Unloading plugins: {plugins}")
799
+
800
+ unload_custom_actions(plugins, debug=debug)
801
+ unload_plugin_connectors(plugins, debug=debug)
802
+
803
+ module_prefix = f"{PLUGINS_RESOURCES_PATH.stem}."
804
+ loaded_modules = [mod_name for mod_name in sys.modules if mod_name.startswith(module_prefix)]
805
+
806
+ for plugin_name in plugins:
807
+ for mod_name in loaded_modules:
808
+ if mod_name[len(PLUGINS_RESOURCES_PATH.stem):].startswith(plugin_name):
809
+ _ = sys.modules.pop(mod_name, None)
810
+
811
+ ### Unload sync hooks.
812
+ _ = _pre_sync_hooks.pop(plugin_name, None)
813
+ _ = _post_sync_hooks.pop(plugin_name, None)
814
+
815
+ ### Unload API endpoints and pages.
816
+ _ = _dash_plugins.pop(plugin_name, None)
817
+ web_page_funcs = _plugins_web_pages.pop(plugin_name, None) or []
818
+ page_groups_to_pop = []
819
+ for page_group, page_functions in _plugin_endpoints_to_pages.items():
820
+ page_functions_to_pop = [
821
+ page_str
822
+ for page_str, page_payload in page_functions.items()
823
+ if page_payload.get('function', None) in web_page_funcs
824
+ ]
825
+ for page_str in page_functions_to_pop:
826
+ page_functions.pop(page_str, None)
827
+ if not page_functions:
828
+ page_groups_to_pop.append(page_group)
829
+
830
+ for page_group in page_groups_to_pop:
831
+ _plugin_endpoints_to_pages.pop(page_group, None)
832
+
833
+ ### Remove all but injected symlinks.
834
+ if remove_symlinks:
835
+ dir_symlink_path = PLUGINS_RESOURCES_PATH / plugin_name
836
+ dir_symlink_injected_path = PLUGINS_INJECTED_RESOURCES_PATH / plugin_name
837
+ file_symlink_path = PLUGINS_RESOURCES_PATH / f"{plugin_name}.py"
838
+ file_symlink_injected_path = PLUGINS_INJECTED_RESOURCES_PATH / f"{plugin_name}.py"
839
+
840
+ try:
841
+ if dir_symlink_path.exists() and not dir_symlink_injected_path.exists():
842
+ dir_symlink_path.unlink()
843
+ except Exception:
844
+ pass
845
+
846
+ try:
847
+ if file_symlink_path.exists() and not file_symlink_injected_path.exists():
848
+ file_symlink_path.unlink()
849
+ except Exception:
850
+ pass
681
851
 
682
852
 
683
853
  def reload_plugins(plugins: Optional[List[str]] = None, debug: bool = False) -> None:
@@ -690,19 +860,11 @@ def reload_plugins(plugins: Optional[List[str]] = None, debug: bool = False) ->
690
860
  The plugins to reload. `None` will reload all plugins.
691
861
 
692
862
  """
693
- import sys
694
- if debug:
695
- from meerschaum.utils.debug import dprint
696
-
697
- if not plugins:
698
- plugins = get_plugins_names()
699
- for plugin_name in plugins:
700
- if debug:
701
- dprint(f"Reloading plugin '{plugin_name}'...")
702
- mod_name = 'plugins.' + str(plugin_name)
703
- if mod_name in sys.modules:
704
- del sys.modules[mod_name]
705
- load_plugins(debug=debug)
863
+ global _synced_symlinks
864
+ unload_plugins(plugins, debug=debug)
865
+ _synced_symlinks = False
866
+ sync_plugins_symlinks(debug=debug)
867
+ load_plugins(skip_if_loaded=False, debug=debug)
706
868
 
707
869
 
708
870
  def get_plugins(*to_load, try_import: bool = True) -> Union[Tuple[Plugin], Plugin]:
@@ -722,7 +884,8 @@ def get_plugins(*to_load, try_import: bool = True) -> Union[Tuple[Plugin], Plugi
722
884
  import os
723
885
  sync_plugins_symlinks()
724
886
  _plugins = [
725
- Plugin(name) for name in (
887
+ Plugin(name)
888
+ for name in (
726
889
  to_load or [
727
890
  (
728
891
  name if (PLUGINS_RESOURCES_PATH / name).is_dir()
@@ -782,8 +945,8 @@ def add_plugin_argument(*args, **kwargs) -> None:
782
945
  >>>
783
946
  """
784
947
  from meerschaum._internal.arguments._parser import groups, _seen_plugin_args, parser
785
- from meerschaum.utils.warnings import warn, error
786
- _parent_plugin_name = _get_parent_plugin(2)
948
+ from meerschaum.utils.warnings import warn
949
+ _parent_plugin_name = _get_parent_plugin()
787
950
  title = f"Plugin '{_parent_plugin_name}' options" if _parent_plugin_name else 'Custom options'
788
951
  group_key = 'plugin_' + (_parent_plugin_name or '')
789
952
  if group_key not in groups:
@@ -799,12 +962,62 @@ def add_plugin_argument(*args, **kwargs) -> None:
799
962
  warn(e)
800
963
 
801
964
 
802
- def _get_parent_plugin(stacklevel: int = 1) -> Union[str, None]:
965
+ def inject_plugin_path(
966
+ plugin_path: pathlib.Path,
967
+ plugins_resources_path: Optional[pathlib.Path] = None) -> None:
968
+ """
969
+ Inject a plugin as a symlink into the internal `plugins` directory.
970
+
971
+ Parameters
972
+ ----------
973
+ plugin_path: pathlib.Path
974
+ The path to the plugin's source module.
975
+ """
976
+ from meerschaum.utils.misc import make_symlink
977
+ if plugins_resources_path is None:
978
+ from meerschaum.config.paths import PLUGINS_RESOURCES_PATH, PLUGINS_INJECTED_RESOURCES_PATH
979
+ plugins_resources_path = PLUGINS_RESOURCES_PATH
980
+ plugins_injected_resources_path = PLUGINS_INJECTED_RESOURCES_PATH
981
+ else:
982
+ plugins_injected_resources_path = plugins_resources_path / '.injected'
983
+
984
+ if plugin_path.is_dir():
985
+ plugin_name = plugin_path.name
986
+ dest_path = plugins_resources_path / plugin_name
987
+ injected_path = plugins_injected_resources_path / plugin_name
988
+ elif plugin_path.name == '__init__.py':
989
+ plugin_name = plugin_path.parent.name
990
+ dest_path = plugins_resources_path / plugin_name
991
+ injected_path = plugins_injected_resources_path / plugin_name
992
+ elif plugin_path.name.endswith('.py'):
993
+ plugin_name = plugin_path.name[:(-1 * len('.py'))]
994
+ dest_path = plugins_resources_path / plugin_path.name
995
+ injected_path = plugins_injected_resources_path / plugin_path.name
996
+ else:
997
+ raise ValueError(f"Cannot deduce plugin name from path '{plugin_path}'.")
998
+
999
+ _injected_plugin_symlinks.add(dest_path)
1000
+ make_symlink(plugin_path, dest_path)
1001
+ make_symlink(plugin_path, injected_path)
1002
+
1003
+
1004
+ def _get_parent_plugin(stacklevel: Union[int, Tuple[int, ...]] = (1, 2, 3, 4)) -> Union[str, None]:
803
1005
  """If this function is called from outside a Meerschaum plugin, it will return None."""
804
- import inspect, re
805
- try:
806
- parent_globals = inspect.stack()[stacklevel][0].f_globals
807
- parent_file = parent_globals.get('__file__', '')
808
- return parent_globals['__name__'].replace('plugins.', '').split('.')[0]
809
- except Exception as e:
810
- return None
1006
+ import inspect
1007
+ if not isinstance(stacklevel, tuple):
1008
+ stacklevel = (stacklevel,)
1009
+
1010
+ for _level in stacklevel:
1011
+ try:
1012
+ parent_globals = inspect.stack()[_level][0].f_globals
1013
+ global_name = parent_globals.get('__name__', '')
1014
+ if global_name.startswith('meerschaum.'):
1015
+ continue
1016
+ plugin_name = global_name.replace('plugins.', '').split('.')[0]
1017
+ if plugin_name.startswith('_') or plugin_name == 'importlib':
1018
+ continue
1019
+ return plugin_name
1020
+ except Exception:
1021
+ continue
1022
+
1023
+ return None