meerschaum 2.3.5.dev0__py3-none-any.whl → 2.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/_internal/arguments/__init__.py +2 -1
- meerschaum/_internal/arguments/_parse_arguments.py +88 -12
- meerschaum/_internal/docs/index.py +3 -2
- meerschaum/_internal/entry.py +42 -20
- meerschaum/_internal/shell/Shell.py +38 -44
- meerschaum/_internal/term/TermPageHandler.py +2 -3
- meerschaum/_internal/term/__init__.py +13 -11
- meerschaum/actions/api.py +26 -23
- meerschaum/actions/bootstrap.py +38 -11
- meerschaum/actions/copy.py +3 -3
- meerschaum/actions/delete.py +4 -1
- meerschaum/actions/register.py +1 -3
- meerschaum/actions/stack.py +24 -19
- meerschaum/actions/start.py +41 -41
- meerschaum/actions/sync.py +53 -52
- meerschaum/api/__init__.py +48 -14
- meerschaum/api/_events.py +26 -17
- meerschaum/api/_oauth2.py +2 -2
- meerschaum/api/_websockets.py +5 -4
- meerschaum/api/dash/__init__.py +7 -16
- meerschaum/api/dash/callbacks/__init__.py +1 -0
- meerschaum/api/dash/callbacks/dashboard.py +52 -58
- meerschaum/api/dash/callbacks/jobs.py +15 -16
- meerschaum/api/dash/callbacks/login.py +16 -10
- meerschaum/api/dash/callbacks/pipes.py +41 -0
- meerschaum/api/dash/callbacks/plugins.py +1 -1
- meerschaum/api/dash/callbacks/register.py +15 -11
- meerschaum/api/dash/components.py +54 -59
- meerschaum/api/dash/jobs.py +5 -9
- meerschaum/api/dash/pages/__init__.py +1 -0
- meerschaum/api/dash/pages/pipes.py +19 -0
- meerschaum/api/dash/pipes.py +86 -58
- meerschaum/api/dash/plugins.py +6 -4
- meerschaum/api/dash/sessions.py +176 -0
- meerschaum/api/dash/users.py +3 -41
- meerschaum/api/dash/webterm.py +12 -17
- meerschaum/api/resources/static/js/terminado.js +1 -1
- meerschaum/api/routes/_actions.py +4 -118
- meerschaum/api/routes/_jobs.py +45 -24
- meerschaum/api/routes/_login.py +4 -4
- meerschaum/api/routes/_pipes.py +3 -3
- meerschaum/api/routes/_webterm.py +5 -6
- meerschaum/config/_default.py +15 -3
- meerschaum/config/_version.py +1 -1
- meerschaum/config/stack/__init__.py +64 -21
- meerschaum/config/static/__init__.py +6 -0
- meerschaum/connectors/{Connector.py → _Connector.py} +19 -13
- meerschaum/connectors/__init__.py +24 -14
- meerschaum/connectors/api/{APIConnector.py → _APIConnector.py} +3 -1
- meerschaum/connectors/api/__init__.py +2 -1
- meerschaum/connectors/api/_actions.py +22 -36
- meerschaum/connectors/api/_jobs.py +1 -0
- meerschaum/connectors/parse.py +18 -16
- meerschaum/connectors/poll.py +30 -24
- meerschaum/connectors/sql/__init__.py +3 -1
- meerschaum/connectors/sql/_pipes.py +172 -197
- meerschaum/connectors/sql/_plugins.py +45 -43
- meerschaum/connectors/sql/_users.py +46 -38
- meerschaum/connectors/valkey/_ValkeyConnector.py +535 -0
- meerschaum/connectors/valkey/__init__.py +10 -0
- meerschaum/connectors/valkey/_fetch.py +75 -0
- meerschaum/connectors/valkey/_pipes.py +844 -0
- meerschaum/connectors/valkey/_plugins.py +265 -0
- meerschaum/connectors/valkey/_users.py +305 -0
- meerschaum/core/Pipe/__init__.py +3 -0
- meerschaum/core/Pipe/_attributes.py +1 -2
- meerschaum/core/Pipe/_clear.py +16 -13
- meerschaum/core/Pipe/_copy.py +106 -0
- meerschaum/core/Pipe/_data.py +165 -101
- meerschaum/core/Pipe/_drop.py +4 -4
- meerschaum/core/Pipe/_dtypes.py +14 -14
- meerschaum/core/Pipe/_edit.py +15 -14
- meerschaum/core/Pipe/_sync.py +134 -53
- meerschaum/core/Pipe/_verify.py +11 -11
- meerschaum/core/User/_User.py +14 -12
- meerschaum/jobs/_Job.py +27 -14
- meerschaum/jobs/__init__.py +7 -2
- meerschaum/jobs/systemd.py +20 -8
- meerschaum/plugins/_Plugin.py +17 -13
- meerschaum/utils/_get_pipes.py +14 -20
- meerschaum/utils/dataframe.py +291 -101
- meerschaum/utils/dtypes/__init__.py +31 -6
- meerschaum/utils/dtypes/sql.py +4 -4
- meerschaum/utils/formatting/_shell.py +5 -6
- meerschaum/utils/misc.py +3 -3
- meerschaum/utils/packages/__init__.py +14 -9
- meerschaum/utils/packages/_packages.py +2 -0
- meerschaum/utils/prompt.py +1 -1
- meerschaum/utils/schedule.py +1 -0
- {meerschaum-2.3.5.dev0.dist-info → meerschaum-2.4.0.dist-info}/METADATA +7 -1
- {meerschaum-2.3.5.dev0.dist-info → meerschaum-2.4.0.dist-info}/RECORD +98 -89
- {meerschaum-2.3.5.dev0.dist-info → meerschaum-2.4.0.dist-info}/WHEEL +1 -1
- meerschaum/api/dash/actions.py +0 -255
- /meerschaum/connectors/sql/{SQLConnector.py → _SQLConnector.py} +0 -0
- {meerschaum-2.3.5.dev0.dist-info → meerschaum-2.4.0.dist-info}/LICENSE +0 -0
- {meerschaum-2.3.5.dev0.dist-info → meerschaum-2.4.0.dist-info}/NOTICE +0 -0
- {meerschaum-2.3.5.dev0.dist-info → meerschaum-2.4.0.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.3.5.dev0.dist-info → meerschaum-2.4.0.dist-info}/top_level.txt +0 -0
- {meerschaum-2.3.5.dev0.dist-info → meerschaum-2.4.0.dist-info}/zip-safe +0 -0
meerschaum/api/dash/actions.py
DELETED
@@ -1,255 +0,0 @@
|
|
1
|
-
#! /usr/bin/env python
|
2
|
-
# -*- coding: utf-8 -*-
|
3
|
-
# vim:fenc=utf-8
|
4
|
-
|
5
|
-
"""
|
6
|
-
Execute actions via the web interface.
|
7
|
-
"""
|
8
|
-
|
9
|
-
from __future__ import annotations
|
10
|
-
import platform, sys, io, os, shlex, time, copy
|
11
|
-
from dash.exceptions import PreventUpdate
|
12
|
-
from meerschaum.utils.threading import Thread
|
13
|
-
from meerschaum.utils.typing import SuccessTuple, Tuple, Dict, Any, WebState
|
14
|
-
from meerschaum.utils.packages import attempt_import, import_html, import_dcc
|
15
|
-
from meerschaum.utils.misc import remove_ansi
|
16
|
-
from meerschaum.actions import actions, get_shell
|
17
|
-
from meerschaum.api import debug, CHECK_UPDATE
|
18
|
-
from meerschaum.api.dash import (
|
19
|
-
running_jobs, stopped_jobs, running_monitors, stopped_monitors, active_sessions
|
20
|
-
)
|
21
|
-
from meerschaum.api import get_api_connector
|
22
|
-
from meerschaum.api.dash.connectors import get_web_connector
|
23
|
-
from meerschaum.api.dash.components import alert_from_success_tuple, console_div
|
24
|
-
from meerschaum.api.dash.webterm import get_webterm
|
25
|
-
from meerschaum.api.dash.pipes import pipes_from_state, keys_from_state
|
26
|
-
from meerschaum.api.dash.websockets import ws_send
|
27
|
-
from meerschaum.api._websockets import websockets
|
28
|
-
html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHECK_UPDATE)
|
29
|
-
from meerschaum.config import get_config
|
30
|
-
from meerschaum.core import User
|
31
|
-
|
32
|
-
def execute_action(state: WebState):
|
33
|
-
"""
|
34
|
-
Execute a Meerschaum action and capture its output.
|
35
|
-
Format the output as an HTML `pre` object, and return a list of Alert objects.
|
36
|
-
|
37
|
-
Parameters
|
38
|
-
----------
|
39
|
-
state: WebState :
|
40
|
-
The state of the dashboard.
|
41
|
-
|
42
|
-
Returns
|
43
|
-
-------
|
44
|
-
A tuple of lists.
|
45
|
-
The first list contains the children of the output div.
|
46
|
-
The second contains alerts.
|
47
|
-
|
48
|
-
"""
|
49
|
-
global running_jobs, stopped_jobs, running_monitors, stopped_monitors
|
50
|
-
action, subaction, subaction_options, subaction_hidden, additional_subaction_text, flags = (
|
51
|
-
state['action-dropdown.value'],
|
52
|
-
state['subaction-dropdown.value'],
|
53
|
-
state['subaction-dropdown.options'],
|
54
|
-
state['subaction-dropdown-div.hidden'],
|
55
|
-
state['subaction-dropdown-text.value'],
|
56
|
-
state['flags-dropdown.value'],
|
57
|
-
)
|
58
|
-
if action is None:
|
59
|
-
return [], []
|
60
|
-
if subaction is None or subaction_hidden:
|
61
|
-
subaction = None
|
62
|
-
### If the current value is not a member of the options, choose the first element (visible).
|
63
|
-
elif subaction not in [o['value'] for o in subaction_options]:
|
64
|
-
subaction = subaction_options[0]['value']
|
65
|
-
if additional_subaction_text is None:
|
66
|
-
additional_subaction_text = ''
|
67
|
-
if flags is None:
|
68
|
-
flags = []
|
69
|
-
|
70
|
-
### TODO add direct Input to parse if active_tab is 'text'
|
71
|
-
|
72
|
-
session_id = state['session-store.data'].get('session-id', None)
|
73
|
-
|
74
|
-
### Check if actions are permitted by non-admin users.
|
75
|
-
permissions = get_config('system', 'api', 'permissions')
|
76
|
-
allow_non_admin = permissions.get('actions', {}).get('non_admin', False)
|
77
|
-
if not allow_non_admin:
|
78
|
-
username = active_sessions.get(session_id, {}).get('username', None)
|
79
|
-
user = User(username, instance=get_api_connector())
|
80
|
-
user_type = get_api_connector().get_user_type(user, debug=debug)
|
81
|
-
if user_type != 'admin':
|
82
|
-
msg = (
|
83
|
-
f"User '{username}' is not authorized to perform actions on this instance.\n\n"
|
84
|
-
+ "To allow non-administrator users to execute actions,\n"
|
85
|
-
+ " run the command `mrsm edit config system`.\n\n"
|
86
|
-
+ "Under the keys `api:permissions:actions`, set the value of \n"
|
87
|
-
+ " `non_admin` to `true`."
|
88
|
-
)
|
89
|
-
return (
|
90
|
-
[
|
91
|
-
html.Div(
|
92
|
-
html.Pre(msg, id='console-pre'),
|
93
|
-
id = 'console-div'
|
94
|
-
),
|
95
|
-
],
|
96
|
-
[alert_from_success_tuple(
|
97
|
-
(False, "Actions are not allowed on this Meerschaum instance.")
|
98
|
-
)]
|
99
|
-
)
|
100
|
-
|
101
|
-
|
102
|
-
subactions = ([subaction] if subaction else []) + shlex.split(additional_subaction_text)
|
103
|
-
|
104
|
-
keywords = {f : True for f in flags}
|
105
|
-
keywords['debug'] = keywords.get('debug', debug)
|
106
|
-
_ck, _mk, _lk, _params = keys_from_state(state, with_params=True)
|
107
|
-
keywords['connector_keys'], keywords['metric_keys'], keywords['location_keys'] = (
|
108
|
-
_ck, _mk, _lk,
|
109
|
-
)
|
110
|
-
keywords['params'] = _params
|
111
|
-
keywords['mrsm_instance'] = get_web_connector(state)
|
112
|
-
|
113
|
-
def do_action() -> SuccessTuple:
|
114
|
-
if sys.stdout is not cap:
|
115
|
-
stdout = sys.stdout
|
116
|
-
sys.stdout = cap
|
117
|
-
try:
|
118
|
-
success_tuple = actions[action](action=subactions, **keywords)
|
119
|
-
except Exception as e:
|
120
|
-
success_tuple = False, str(e)
|
121
|
-
except KeyboardInterrupt:
|
122
|
-
success_tuple = False, f"Action '{action}' was manually cancelled."
|
123
|
-
stopped_jobs[session_id] = running_jobs.pop(session_id)
|
124
|
-
stopped_monitors[session_id] = running_monitors.pop(session_id)
|
125
|
-
return success_tuple
|
126
|
-
|
127
|
-
cap = io.StringIO()
|
128
|
-
_sentinel = object()
|
129
|
-
def monitor_and_send_stdout():
|
130
|
-
cap_buffer = ''
|
131
|
-
while True:
|
132
|
-
if not (sys.stdout) is cap:
|
133
|
-
sys.stdout = cap
|
134
|
-
if session_id not in running_jobs:
|
135
|
-
text = cap.getvalue()
|
136
|
-
print('Stopping monitoring and sending final message.', file=sys.stderr)
|
137
|
-
ws_send(text, session_id)
|
138
|
-
break
|
139
|
-
text = cap.getvalue()
|
140
|
-
if not text or len(text) == len(cap_buffer):
|
141
|
-
continue
|
142
|
-
cap_buffer = text
|
143
|
-
ws_send(text, session_id)
|
144
|
-
### So we don't overwhelm the client.
|
145
|
-
time.sleep(0.01)
|
146
|
-
|
147
|
-
def use_stringio():
|
148
|
-
try:
|
149
|
-
LINES, COLUMNS = (
|
150
|
-
os.environ.get('LINES', str(os.get_terminal_size().lines)),
|
151
|
-
os.environ.get('COLUMNS', str(os.get_terminal_size().columns)),
|
152
|
-
)
|
153
|
-
except OSError:
|
154
|
-
LINES, COLUMNS = '120', '100'
|
155
|
-
os.environ['LINES'], os.environ['COLUMNS'] = '120', '100'
|
156
|
-
stdout = sys.stdout
|
157
|
-
success_tuple = use_process()
|
158
|
-
os.environ['LINES'], os.environ['COLUMNS'] = LINES, COLUMNS
|
159
|
-
text = cap.getvalue()
|
160
|
-
return text, success_tuple
|
161
|
-
|
162
|
-
|
163
|
-
def use_process():
|
164
|
-
from meerschaum.utils.packages import run_python_package
|
165
|
-
from meerschaum._internal.arguments._parse_arguments import parse_dict_to_sysargs
|
166
|
-
max_buffer_size = 10000
|
167
|
-
line_buffer = ''
|
168
|
-
def send_line(line: str):
|
169
|
-
nonlocal line_buffer
|
170
|
-
line_buffer += line
|
171
|
-
buff_start_idx = max(len(line_buffer) - max_buffer_size, 0)
|
172
|
-
line_buffer = line_buffer[buff_start_idx:]
|
173
|
-
ws_send(line_buffer, session_id)
|
174
|
-
|
175
|
-
def do_process():
|
176
|
-
keywords['action'] = [action] + subactions
|
177
|
-
_sysargs = parse_dict_to_sysargs(keywords)
|
178
|
-
|
179
|
-
### Don't forget to pass the existing environment
|
180
|
-
### in case MRSM_CONFIG and MRSM_PATCH are set.
|
181
|
-
try:
|
182
|
-
_env = copy.deepcopy(os.environ)
|
183
|
-
except Exception as e:
|
184
|
-
_env = {}
|
185
|
-
_env.update({'LINES': '120', 'COLUMNS': '100'})
|
186
|
-
|
187
|
-
process = run_python_package(
|
188
|
-
'meerschaum', _sysargs,
|
189
|
-
line_callback = send_line,
|
190
|
-
env = _env,
|
191
|
-
foreground = False,
|
192
|
-
universal_newlines = True,
|
193
|
-
as_proc = True,
|
194
|
-
capture_output = False,
|
195
|
-
### Store the `subprocess.Popen` process in case we need to terminate it later.
|
196
|
-
store_proc_dict = running_jobs[session_id],
|
197
|
-
venv = None,
|
198
|
-
debug = debug,
|
199
|
-
)
|
200
|
-
action_thread = Thread(target=do_process)
|
201
|
-
running_jobs[session_id] = {'parent_thread': action_thread,}
|
202
|
-
action_thread.start()
|
203
|
-
return True, "Success"
|
204
|
-
|
205
|
-
|
206
|
-
def use_thread():
|
207
|
-
action_thread = Thread(target=do_action, daemon=True)
|
208
|
-
monitor_thread = Thread(target=monitor_and_send_stdout)
|
209
|
-
running_jobs[session_id] = action_thread
|
210
|
-
running_monitors[session_id] = monitor_thread
|
211
|
-
monitor_thread.start()
|
212
|
-
action_thread.start()
|
213
|
-
success_tuple = True, 'Success'
|
214
|
-
return success_tuple
|
215
|
-
|
216
|
-
text, success_tuple = use_stringio()
|
217
|
-
|
218
|
-
return (
|
219
|
-
[
|
220
|
-
html.Div(
|
221
|
-
html.Pre(text, id='console-pre'),
|
222
|
-
id = 'console-div'
|
223
|
-
),
|
224
|
-
],
|
225
|
-
[alert_from_success_tuple(success_tuple)]
|
226
|
-
)
|
227
|
-
|
228
|
-
|
229
|
-
def stop_action(state: WebState):
|
230
|
-
"""
|
231
|
-
Kill a running action.
|
232
|
-
|
233
|
-
Parameters
|
234
|
-
----------
|
235
|
-
state: WebState:
|
236
|
-
The state of the dashboard.
|
237
|
-
|
238
|
-
Returns
|
239
|
-
-------
|
240
|
-
A tuple of children and alerts.
|
241
|
-
"""
|
242
|
-
global running_jobs
|
243
|
-
session_id = state['session-store.data'].get('session-id', None)
|
244
|
-
thread = running_jobs.get(session_id, {}).get('parent_thread', None)
|
245
|
-
proc = running_jobs.get(session_id, {}).get('child_process', None)
|
246
|
-
if proc is not None and proc.poll() is None:
|
247
|
-
proc.terminate()
|
248
|
-
thread.join()
|
249
|
-
success_tuple = True, "Success"
|
250
|
-
webterm_iframe, webterm_alerts = get_webterm(state)
|
251
|
-
console_to_return = console_div if webterm_alerts else webterm_iframe
|
252
|
-
return (
|
253
|
-
[console_to_return],
|
254
|
-
[alert_from_success_tuple(success_tuple)]
|
255
|
-
)
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|