meerschaum 2.3.5.dev0__py3-none-any.whl → 2.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. meerschaum/_internal/arguments/__init__.py +2 -1
  2. meerschaum/_internal/arguments/_parse_arguments.py +88 -12
  3. meerschaum/_internal/docs/index.py +3 -2
  4. meerschaum/_internal/entry.py +42 -20
  5. meerschaum/_internal/shell/Shell.py +38 -44
  6. meerschaum/_internal/term/TermPageHandler.py +2 -3
  7. meerschaum/_internal/term/__init__.py +13 -11
  8. meerschaum/actions/api.py +26 -23
  9. meerschaum/actions/bootstrap.py +38 -11
  10. meerschaum/actions/copy.py +3 -3
  11. meerschaum/actions/delete.py +4 -1
  12. meerschaum/actions/register.py +1 -3
  13. meerschaum/actions/stack.py +24 -19
  14. meerschaum/actions/start.py +41 -41
  15. meerschaum/actions/sync.py +53 -52
  16. meerschaum/api/__init__.py +48 -14
  17. meerschaum/api/_events.py +26 -17
  18. meerschaum/api/_oauth2.py +2 -2
  19. meerschaum/api/_websockets.py +5 -4
  20. meerschaum/api/dash/__init__.py +7 -16
  21. meerschaum/api/dash/callbacks/__init__.py +1 -0
  22. meerschaum/api/dash/callbacks/dashboard.py +52 -58
  23. meerschaum/api/dash/callbacks/jobs.py +15 -16
  24. meerschaum/api/dash/callbacks/login.py +16 -10
  25. meerschaum/api/dash/callbacks/pipes.py +41 -0
  26. meerschaum/api/dash/callbacks/plugins.py +1 -1
  27. meerschaum/api/dash/callbacks/register.py +15 -11
  28. meerschaum/api/dash/components.py +54 -59
  29. meerschaum/api/dash/jobs.py +5 -9
  30. meerschaum/api/dash/pages/__init__.py +1 -0
  31. meerschaum/api/dash/pages/pipes.py +19 -0
  32. meerschaum/api/dash/pipes.py +86 -58
  33. meerschaum/api/dash/plugins.py +6 -4
  34. meerschaum/api/dash/sessions.py +176 -0
  35. meerschaum/api/dash/users.py +3 -41
  36. meerschaum/api/dash/webterm.py +12 -17
  37. meerschaum/api/resources/static/js/terminado.js +1 -1
  38. meerschaum/api/routes/_actions.py +4 -118
  39. meerschaum/api/routes/_jobs.py +45 -24
  40. meerschaum/api/routes/_login.py +4 -4
  41. meerschaum/api/routes/_pipes.py +3 -3
  42. meerschaum/api/routes/_webterm.py +5 -6
  43. meerschaum/config/_default.py +15 -3
  44. meerschaum/config/_version.py +1 -1
  45. meerschaum/config/stack/__init__.py +64 -21
  46. meerschaum/config/static/__init__.py +6 -0
  47. meerschaum/connectors/{Connector.py → _Connector.py} +19 -13
  48. meerschaum/connectors/__init__.py +24 -14
  49. meerschaum/connectors/api/{APIConnector.py → _APIConnector.py} +3 -1
  50. meerschaum/connectors/api/__init__.py +2 -1
  51. meerschaum/connectors/api/_actions.py +22 -36
  52. meerschaum/connectors/api/_jobs.py +1 -0
  53. meerschaum/connectors/parse.py +18 -16
  54. meerschaum/connectors/poll.py +30 -24
  55. meerschaum/connectors/sql/__init__.py +3 -1
  56. meerschaum/connectors/sql/_pipes.py +172 -197
  57. meerschaum/connectors/sql/_plugins.py +45 -43
  58. meerschaum/connectors/sql/_users.py +46 -38
  59. meerschaum/connectors/valkey/_ValkeyConnector.py +535 -0
  60. meerschaum/connectors/valkey/__init__.py +10 -0
  61. meerschaum/connectors/valkey/_fetch.py +75 -0
  62. meerschaum/connectors/valkey/_pipes.py +844 -0
  63. meerschaum/connectors/valkey/_plugins.py +265 -0
  64. meerschaum/connectors/valkey/_users.py +305 -0
  65. meerschaum/core/Pipe/__init__.py +3 -0
  66. meerschaum/core/Pipe/_attributes.py +1 -2
  67. meerschaum/core/Pipe/_clear.py +16 -13
  68. meerschaum/core/Pipe/_copy.py +106 -0
  69. meerschaum/core/Pipe/_data.py +165 -101
  70. meerschaum/core/Pipe/_drop.py +4 -4
  71. meerschaum/core/Pipe/_dtypes.py +14 -14
  72. meerschaum/core/Pipe/_edit.py +15 -14
  73. meerschaum/core/Pipe/_sync.py +134 -53
  74. meerschaum/core/Pipe/_verify.py +11 -11
  75. meerschaum/core/User/_User.py +14 -12
  76. meerschaum/jobs/_Job.py +27 -14
  77. meerschaum/jobs/__init__.py +7 -2
  78. meerschaum/jobs/systemd.py +20 -8
  79. meerschaum/plugins/_Plugin.py +17 -13
  80. meerschaum/utils/_get_pipes.py +14 -20
  81. meerschaum/utils/dataframe.py +291 -101
  82. meerschaum/utils/dtypes/__init__.py +31 -6
  83. meerschaum/utils/dtypes/sql.py +4 -4
  84. meerschaum/utils/formatting/_shell.py +5 -6
  85. meerschaum/utils/misc.py +3 -3
  86. meerschaum/utils/packages/__init__.py +14 -9
  87. meerschaum/utils/packages/_packages.py +2 -0
  88. meerschaum/utils/prompt.py +1 -1
  89. meerschaum/utils/schedule.py +1 -0
  90. {meerschaum-2.3.5.dev0.dist-info → meerschaum-2.4.0.dist-info}/METADATA +7 -1
  91. {meerschaum-2.3.5.dev0.dist-info → meerschaum-2.4.0.dist-info}/RECORD +98 -89
  92. {meerschaum-2.3.5.dev0.dist-info → meerschaum-2.4.0.dist-info}/WHEEL +1 -1
  93. meerschaum/api/dash/actions.py +0 -255
  94. /meerschaum/connectors/sql/{SQLConnector.py → _SQLConnector.py} +0 -0
  95. {meerschaum-2.3.5.dev0.dist-info → meerschaum-2.4.0.dist-info}/LICENSE +0 -0
  96. {meerschaum-2.3.5.dev0.dist-info → meerschaum-2.4.0.dist-info}/NOTICE +0 -0
  97. {meerschaum-2.3.5.dev0.dist-info → meerschaum-2.4.0.dist-info}/entry_points.txt +0 -0
  98. {meerschaum-2.3.5.dev0.dist-info → meerschaum-2.4.0.dist-info}/top_level.txt +0 -0
  99. {meerschaum-2.3.5.dev0.dist-info → meerschaum-2.4.0.dist-info}/zip-safe +0 -0
@@ -1,255 +0,0 @@
1
- #! /usr/bin/env python
2
- # -*- coding: utf-8 -*-
3
- # vim:fenc=utf-8
4
-
5
- """
6
- Execute actions via the web interface.
7
- """
8
-
9
- from __future__ import annotations
10
- import platform, sys, io, os, shlex, time, copy
11
- from dash.exceptions import PreventUpdate
12
- from meerschaum.utils.threading import Thread
13
- from meerschaum.utils.typing import SuccessTuple, Tuple, Dict, Any, WebState
14
- from meerschaum.utils.packages import attempt_import, import_html, import_dcc
15
- from meerschaum.utils.misc import remove_ansi
16
- from meerschaum.actions import actions, get_shell
17
- from meerschaum.api import debug, CHECK_UPDATE
18
- from meerschaum.api.dash import (
19
- running_jobs, stopped_jobs, running_monitors, stopped_monitors, active_sessions
20
- )
21
- from meerschaum.api import get_api_connector
22
- from meerschaum.api.dash.connectors import get_web_connector
23
- from meerschaum.api.dash.components import alert_from_success_tuple, console_div
24
- from meerschaum.api.dash.webterm import get_webterm
25
- from meerschaum.api.dash.pipes import pipes_from_state, keys_from_state
26
- from meerschaum.api.dash.websockets import ws_send
27
- from meerschaum.api._websockets import websockets
28
- html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHECK_UPDATE)
29
- from meerschaum.config import get_config
30
- from meerschaum.core import User
31
-
32
- def execute_action(state: WebState):
33
- """
34
- Execute a Meerschaum action and capture its output.
35
- Format the output as an HTML `pre` object, and return a list of Alert objects.
36
-
37
- Parameters
38
- ----------
39
- state: WebState :
40
- The state of the dashboard.
41
-
42
- Returns
43
- -------
44
- A tuple of lists.
45
- The first list contains the children of the output div.
46
- The second contains alerts.
47
-
48
- """
49
- global running_jobs, stopped_jobs, running_monitors, stopped_monitors
50
- action, subaction, subaction_options, subaction_hidden, additional_subaction_text, flags = (
51
- state['action-dropdown.value'],
52
- state['subaction-dropdown.value'],
53
- state['subaction-dropdown.options'],
54
- state['subaction-dropdown-div.hidden'],
55
- state['subaction-dropdown-text.value'],
56
- state['flags-dropdown.value'],
57
- )
58
- if action is None:
59
- return [], []
60
- if subaction is None or subaction_hidden:
61
- subaction = None
62
- ### If the current value is not a member of the options, choose the first element (visible).
63
- elif subaction not in [o['value'] for o in subaction_options]:
64
- subaction = subaction_options[0]['value']
65
- if additional_subaction_text is None:
66
- additional_subaction_text = ''
67
- if flags is None:
68
- flags = []
69
-
70
- ### TODO add direct Input to parse if active_tab is 'text'
71
-
72
- session_id = state['session-store.data'].get('session-id', None)
73
-
74
- ### Check if actions are permitted by non-admin users.
75
- permissions = get_config('system', 'api', 'permissions')
76
- allow_non_admin = permissions.get('actions', {}).get('non_admin', False)
77
- if not allow_non_admin:
78
- username = active_sessions.get(session_id, {}).get('username', None)
79
- user = User(username, instance=get_api_connector())
80
- user_type = get_api_connector().get_user_type(user, debug=debug)
81
- if user_type != 'admin':
82
- msg = (
83
- f"User '{username}' is not authorized to perform actions on this instance.\n\n"
84
- + "To allow non-administrator users to execute actions,\n"
85
- + " run the command `mrsm edit config system`.\n\n"
86
- + "Under the keys `api:permissions:actions`, set the value of \n"
87
- + " `non_admin` to `true`."
88
- )
89
- return (
90
- [
91
- html.Div(
92
- html.Pre(msg, id='console-pre'),
93
- id = 'console-div'
94
- ),
95
- ],
96
- [alert_from_success_tuple(
97
- (False, "Actions are not allowed on this Meerschaum instance.")
98
- )]
99
- )
100
-
101
-
102
- subactions = ([subaction] if subaction else []) + shlex.split(additional_subaction_text)
103
-
104
- keywords = {f : True for f in flags}
105
- keywords['debug'] = keywords.get('debug', debug)
106
- _ck, _mk, _lk, _params = keys_from_state(state, with_params=True)
107
- keywords['connector_keys'], keywords['metric_keys'], keywords['location_keys'] = (
108
- _ck, _mk, _lk,
109
- )
110
- keywords['params'] = _params
111
- keywords['mrsm_instance'] = get_web_connector(state)
112
-
113
- def do_action() -> SuccessTuple:
114
- if sys.stdout is not cap:
115
- stdout = sys.stdout
116
- sys.stdout = cap
117
- try:
118
- success_tuple = actions[action](action=subactions, **keywords)
119
- except Exception as e:
120
- success_tuple = False, str(e)
121
- except KeyboardInterrupt:
122
- success_tuple = False, f"Action '{action}' was manually cancelled."
123
- stopped_jobs[session_id] = running_jobs.pop(session_id)
124
- stopped_monitors[session_id] = running_monitors.pop(session_id)
125
- return success_tuple
126
-
127
- cap = io.StringIO()
128
- _sentinel = object()
129
- def monitor_and_send_stdout():
130
- cap_buffer = ''
131
- while True:
132
- if not (sys.stdout) is cap:
133
- sys.stdout = cap
134
- if session_id not in running_jobs:
135
- text = cap.getvalue()
136
- print('Stopping monitoring and sending final message.', file=sys.stderr)
137
- ws_send(text, session_id)
138
- break
139
- text = cap.getvalue()
140
- if not text or len(text) == len(cap_buffer):
141
- continue
142
- cap_buffer = text
143
- ws_send(text, session_id)
144
- ### So we don't overwhelm the client.
145
- time.sleep(0.01)
146
-
147
- def use_stringio():
148
- try:
149
- LINES, COLUMNS = (
150
- os.environ.get('LINES', str(os.get_terminal_size().lines)),
151
- os.environ.get('COLUMNS', str(os.get_terminal_size().columns)),
152
- )
153
- except OSError:
154
- LINES, COLUMNS = '120', '100'
155
- os.environ['LINES'], os.environ['COLUMNS'] = '120', '100'
156
- stdout = sys.stdout
157
- success_tuple = use_process()
158
- os.environ['LINES'], os.environ['COLUMNS'] = LINES, COLUMNS
159
- text = cap.getvalue()
160
- return text, success_tuple
161
-
162
-
163
- def use_process():
164
- from meerschaum.utils.packages import run_python_package
165
- from meerschaum._internal.arguments._parse_arguments import parse_dict_to_sysargs
166
- max_buffer_size = 10000
167
- line_buffer = ''
168
- def send_line(line: str):
169
- nonlocal line_buffer
170
- line_buffer += line
171
- buff_start_idx = max(len(line_buffer) - max_buffer_size, 0)
172
- line_buffer = line_buffer[buff_start_idx:]
173
- ws_send(line_buffer, session_id)
174
-
175
- def do_process():
176
- keywords['action'] = [action] + subactions
177
- _sysargs = parse_dict_to_sysargs(keywords)
178
-
179
- ### Don't forget to pass the existing environment
180
- ### in case MRSM_CONFIG and MRSM_PATCH are set.
181
- try:
182
- _env = copy.deepcopy(os.environ)
183
- except Exception as e:
184
- _env = {}
185
- _env.update({'LINES': '120', 'COLUMNS': '100'})
186
-
187
- process = run_python_package(
188
- 'meerschaum', _sysargs,
189
- line_callback = send_line,
190
- env = _env,
191
- foreground = False,
192
- universal_newlines = True,
193
- as_proc = True,
194
- capture_output = False,
195
- ### Store the `subprocess.Popen` process in case we need to terminate it later.
196
- store_proc_dict = running_jobs[session_id],
197
- venv = None,
198
- debug = debug,
199
- )
200
- action_thread = Thread(target=do_process)
201
- running_jobs[session_id] = {'parent_thread': action_thread,}
202
- action_thread.start()
203
- return True, "Success"
204
-
205
-
206
- def use_thread():
207
- action_thread = Thread(target=do_action, daemon=True)
208
- monitor_thread = Thread(target=monitor_and_send_stdout)
209
- running_jobs[session_id] = action_thread
210
- running_monitors[session_id] = monitor_thread
211
- monitor_thread.start()
212
- action_thread.start()
213
- success_tuple = True, 'Success'
214
- return success_tuple
215
-
216
- text, success_tuple = use_stringio()
217
-
218
- return (
219
- [
220
- html.Div(
221
- html.Pre(text, id='console-pre'),
222
- id = 'console-div'
223
- ),
224
- ],
225
- [alert_from_success_tuple(success_tuple)]
226
- )
227
-
228
-
229
- def stop_action(state: WebState):
230
- """
231
- Kill a running action.
232
-
233
- Parameters
234
- ----------
235
- state: WebState:
236
- The state of the dashboard.
237
-
238
- Returns
239
- -------
240
- A tuple of children and alerts.
241
- """
242
- global running_jobs
243
- session_id = state['session-store.data'].get('session-id', None)
244
- thread = running_jobs.get(session_id, {}).get('parent_thread', None)
245
- proc = running_jobs.get(session_id, {}).get('child_process', None)
246
- if proc is not None and proc.poll() is None:
247
- proc.terminate()
248
- thread.join()
249
- success_tuple = True, "Success"
250
- webterm_iframe, webterm_alerts = get_webterm(state)
251
- console_to_return = console_div if webterm_alerts else webterm_iframe
252
- return (
253
- [console_to_return],
254
- [alert_from_success_tuple(success_tuple)]
255
- )