meerschaum 2.2.7__py3-none-any.whl → 2.3.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/__init__.py +4 -1
- meerschaum/_internal/arguments/_parser.py +44 -15
- meerschaum/_internal/entry.py +22 -1
- meerschaum/_internal/shell/Shell.py +129 -31
- meerschaum/actions/api.py +12 -12
- meerschaum/actions/attach.py +95 -0
- meerschaum/actions/delete.py +35 -26
- meerschaum/actions/show.py +119 -148
- meerschaum/actions/start.py +85 -75
- meerschaum/actions/stop.py +68 -39
- meerschaum/api/_events.py +18 -1
- meerschaum/api/_oauth2.py +2 -0
- meerschaum/api/_websockets.py +2 -2
- meerschaum/api/dash/jobs.py +5 -2
- meerschaum/api/routes/__init__.py +1 -0
- meerschaum/api/routes/_actions.py +122 -44
- meerschaum/api/routes/_jobs.py +340 -0
- meerschaum/api/routes/_pipes.py +5 -5
- meerschaum/config/_default.py +1 -0
- meerschaum/config/_paths.py +1 -0
- meerschaum/config/_shell.py +8 -3
- meerschaum/config/_version.py +1 -1
- meerschaum/config/static/__init__.py +8 -0
- meerschaum/connectors/__init__.py +9 -11
- meerschaum/connectors/api/APIConnector.py +18 -1
- meerschaum/connectors/api/_actions.py +60 -71
- meerschaum/connectors/api/_jobs.py +260 -0
- meerschaum/connectors/parse.py +23 -7
- meerschaum/plugins/__init__.py +89 -5
- meerschaum/utils/daemon/Daemon.py +255 -30
- meerschaum/utils/daemon/FileDescriptorInterceptor.py +5 -5
- meerschaum/utils/daemon/RotatingFile.py +10 -6
- meerschaum/utils/daemon/StdinFile.py +110 -0
- meerschaum/utils/daemon/__init__.py +13 -7
- meerschaum/utils/formatting/__init__.py +2 -1
- meerschaum/utils/formatting/_jobs.py +83 -54
- meerschaum/utils/formatting/_shell.py +6 -0
- meerschaum/utils/jobs/_Job.py +684 -0
- meerschaum/utils/jobs/__init__.py +245 -0
- meerschaum/utils/misc.py +18 -17
- meerschaum/utils/packages/_packages.py +2 -2
- meerschaum/utils/prompt.py +16 -8
- {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dev1.dist-info}/METADATA +9 -9
- {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dev1.dist-info}/RECORD +50 -44
- {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dev1.dist-info}/WHEEL +1 -1
- {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dev1.dist-info}/LICENSE +0 -0
- {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dev1.dist-info}/NOTICE +0 -0
- {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dev1.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dev1.dist-info}/top_level.txt +0 -0
- {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dev1.dist-info}/zip-safe +0 -0
@@ -7,65 +7,143 @@ Execute Meerschaum Actions via the API
|
|
7
7
|
"""
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
|
-
|
10
|
+
import asyncio
|
11
|
+
import traceback
|
12
|
+
import shlex
|
13
|
+
from functools import partial
|
14
|
+
from datetime import datetime, timezone
|
11
15
|
|
16
|
+
from fastapi import WebSocket, WebSocketDisconnect
|
17
|
+
|
18
|
+
from meerschaum.utils.misc import generate_password
|
19
|
+
from meerschaum.utils.jobs import Job
|
20
|
+
from meerschaum.utils.warnings import warn
|
21
|
+
from meerschaum.utils.typing import SuccessTuple, Union, List, Dict
|
12
22
|
from meerschaum.api import (
|
13
23
|
fastapi, app, endpoints, get_api_connector, debug, manager, private, no_auth
|
14
24
|
)
|
15
25
|
from meerschaum.actions import actions
|
16
26
|
import meerschaum.core
|
27
|
+
from meerschaum.config import get_config
|
28
|
+
from meerschaum._internal.arguments._parse_arguments import parse_dict_to_sysargs, parse_arguments
|
29
|
+
|
17
30
|
actions_endpoint = endpoints['actions']
|
18
31
|
|
32
|
+
def is_user_allowed_to_execute(user) -> SuccessTuple:
|
33
|
+
if user is None:
|
34
|
+
return False, "Could not load user."
|
35
|
+
|
36
|
+
if user.type == 'admin':
|
37
|
+
return True, "Success"
|
38
|
+
|
39
|
+
allow_non_admin = get_config(
|
40
|
+
'system', 'api', 'permissions', 'actions', 'non_admin', patch=True
|
41
|
+
)
|
42
|
+
if not allow_non_admin:
|
43
|
+
return False, (
|
44
|
+
"The administrator for this server has not allowed users to perform actions.\n\n"
|
45
|
+
+ "Please contact the system administrator, or if you are running this server, "
|
46
|
+
+ "open the configuration file with `edit config system` "
|
47
|
+
+ "and search for 'permissions'. "
|
48
|
+
+ "\nUnder the keys 'api:permissions:actions', "
|
49
|
+
+ "you can allow non-admin users to perform actions."
|
50
|
+
)
|
51
|
+
|
52
|
+
return True, "Success"
|
53
|
+
|
54
|
+
|
19
55
|
@app.get(actions_endpoint, tags=['Actions'])
|
20
56
|
def get_actions(
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
57
|
+
curr_user = (
|
58
|
+
fastapi.Depends(manager) if private else None
|
59
|
+
),
|
60
|
+
) -> List[str]:
|
61
|
+
"""
|
62
|
+
Return a list of the available actions.
|
63
|
+
"""
|
25
64
|
return list(actions)
|
26
65
|
|
27
|
-
|
28
|
-
def
|
29
|
-
|
30
|
-
|
31
|
-
curr_user = (
|
32
|
-
fastapi.Depends(manager) if not no_auth else None
|
33
|
-
),
|
34
|
-
) -> SuccessTuple:
|
66
|
+
|
67
|
+
async def notify_client(client, content: str):
|
68
|
+
"""
|
69
|
+
Send a line of text to a client.
|
35
70
|
"""
|
36
|
-
|
71
|
+
try:
|
72
|
+
await client.send_text(content)
|
73
|
+
except WebSocketDisconnect:
|
74
|
+
pass
|
37
75
|
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
76
|
+
_temp_jobs = {}
|
77
|
+
@app.websocket(actions_endpoint + '/ws')
|
78
|
+
async def do_action_websocket(websocket: WebSocket):
|
79
|
+
"""
|
80
|
+
Execute an action and stream the output to the client.
|
81
|
+
"""
|
82
|
+
await websocket.accept()
|
45
83
|
|
46
|
-
|
47
|
-
-------
|
48
|
-
A `SuccessTuple` of success, message.
|
84
|
+
stop_event = asyncio.Event()
|
49
85
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
if not allow_non_admin:
|
57
|
-
return False, (
|
58
|
-
"The administrator for this server has not allowed users to perform actions.\n\n"
|
59
|
-
+ "Please contact the system administrator, or if you are running this server, "
|
60
|
-
+ "open the configuration file with `edit config system` "
|
61
|
-
+ "and search for 'permissions'. "
|
62
|
-
+ "\nUnder the keys 'api:permissions:actions', "
|
63
|
-
+ "you can allow non-admin users to perform actions."
|
86
|
+
async def monitor_logs(job):
|
87
|
+
success, msg = job.start()
|
88
|
+
await job.monitor_logs_async(
|
89
|
+
partial(notify_client, websocket),
|
90
|
+
stop_event=stop_event,
|
91
|
+
stop_on_exit=True,
|
64
92
|
)
|
65
93
|
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
94
|
+
job = None
|
95
|
+
try:
|
96
|
+
token = await websocket.receive_text()
|
97
|
+
user = await manager.get_current_user(token) if not no_auth else None
|
98
|
+
if user is None and not no_auth:
|
99
|
+
raise fastapi.HTTPException(
|
100
|
+
status_code=401,
|
101
|
+
detail="Invalid credentials.",
|
102
|
+
)
|
103
|
+
|
104
|
+
auth_success, auth_msg = is_user_allowed_to_execute(user)
|
105
|
+
auth_payload = {
|
106
|
+
'is_authenticated': auth_success,
|
107
|
+
'timestamp': datetime.now(timezone.utc).isoformat(),
|
108
|
+
}
|
109
|
+
await websocket.send_json(auth_payload)
|
110
|
+
if not auth_success:
|
111
|
+
await websocket.close()
|
112
|
+
|
113
|
+
sysargs = await websocket.receive_json()
|
114
|
+
kwargs = parse_arguments(sysargs)
|
115
|
+
_ = kwargs.pop('executor_keys', None)
|
116
|
+
_ = kwargs.pop('shell', None)
|
117
|
+
sysargs = parse_dict_to_sysargs(kwargs)
|
118
|
+
|
119
|
+
job_name = '.' + generate_password(12)
|
120
|
+
job = Job(
|
121
|
+
job_name,
|
122
|
+
sysargs,
|
123
|
+
_properties={
|
124
|
+
'logs': {
|
125
|
+
'write_timestamps': False,
|
126
|
+
},
|
127
|
+
},
|
128
|
+
)
|
129
|
+
_temp_jobs[job_name] = job
|
130
|
+
monitor_task = asyncio.create_task(monitor_logs(job))
|
131
|
+
await monitor_task
|
132
|
+
try:
|
133
|
+
await websocket.close()
|
134
|
+
except RuntimeError:
|
135
|
+
pass
|
136
|
+
except fastapi.HTTPException:
|
137
|
+
await websocket.send_text("Invalid credentials.")
|
138
|
+
await websocket.close()
|
139
|
+
except WebSocketDisconnect:
|
140
|
+
pass
|
141
|
+
except asyncio.CancelledError:
|
142
|
+
pass
|
143
|
+
except Exception:
|
144
|
+
warn(f"Error in logs websocket:\n{traceback.format_exc()}")
|
145
|
+
finally:
|
146
|
+
if job is not None:
|
147
|
+
job.delete()
|
148
|
+
_ = _temp_jobs.pop(job_name, None)
|
149
|
+
stop_event.set()
|
@@ -0,0 +1,340 @@
|
|
1
|
+
#! /usr/bin/env python3
|
2
|
+
# -*- coding: utf-8 -*-
|
3
|
+
# vim:fenc=utf-8
|
4
|
+
|
5
|
+
"""
|
6
|
+
Manage jobs via the Meerschaum API.
|
7
|
+
"""
|
8
|
+
|
9
|
+
from __future__ import annotations
|
10
|
+
|
11
|
+
import os
|
12
|
+
import select
|
13
|
+
import asyncio
|
14
|
+
import traceback
|
15
|
+
from datetime import datetime
|
16
|
+
from collections import defaultdict
|
17
|
+
from functools import partial
|
18
|
+
|
19
|
+
from fastapi import WebSocket, WebSocketDisconnect
|
20
|
+
|
21
|
+
from meerschaum.utils.typing import Dict, Any, SuccessTuple, List, Optional, Union
|
22
|
+
from meerschaum.utils.jobs import get_jobs as _get_jobs, Job, StopMonitoringLogs
|
23
|
+
from meerschaum.utils.warnings import warn
|
24
|
+
|
25
|
+
from meerschaum.api import (
|
26
|
+
fastapi,
|
27
|
+
app,
|
28
|
+
endpoints,
|
29
|
+
manager,
|
30
|
+
debug,
|
31
|
+
no_auth,
|
32
|
+
private,
|
33
|
+
)
|
34
|
+
from meerschaum.config.static import STATIC_CONFIG
|
35
|
+
|
36
|
+
JOBS_STDIN_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stdin_message']
|
37
|
+
|
38
|
+
|
39
|
+
@app.get(endpoints['jobs'], tags=['Jobs'])
|
40
|
+
def get_jobs(
|
41
|
+
curr_user=(
|
42
|
+
fastapi.Depends(manager) if not no_auth else None
|
43
|
+
),
|
44
|
+
) -> Dict[str, Dict[str, Any]]:
|
45
|
+
"""
|
46
|
+
Return metadata about the current jobs.
|
47
|
+
"""
|
48
|
+
jobs = _get_jobs()
|
49
|
+
return {
|
50
|
+
name: {
|
51
|
+
'sysargs': job.sysargs,
|
52
|
+
'result': job.result,
|
53
|
+
'daemon': {
|
54
|
+
'status': job.daemon.status,
|
55
|
+
'pid': job.daemon.pid,
|
56
|
+
'properties': job.daemon.properties,
|
57
|
+
},
|
58
|
+
}
|
59
|
+
for name, job in jobs.items()
|
60
|
+
}
|
61
|
+
|
62
|
+
|
63
|
+
@app.get(endpoints['jobs'] + '/{name}', tags=['Jobs'])
|
64
|
+
def get_job(
|
65
|
+
name: str,
|
66
|
+
curr_user=(
|
67
|
+
fastapi.Depends(manager) if not no_auth else None
|
68
|
+
),
|
69
|
+
) -> Dict[str, Any]:
|
70
|
+
"""
|
71
|
+
Return metadata for a single job.
|
72
|
+
"""
|
73
|
+
job = Job(name)
|
74
|
+
if not job.exists():
|
75
|
+
raise fastapi.HTTPException(
|
76
|
+
status_code=404,
|
77
|
+
detail=f"{job} doesn't exist."
|
78
|
+
)
|
79
|
+
|
80
|
+
return {
|
81
|
+
'sysargs': job.sysargs,
|
82
|
+
'result': job.result,
|
83
|
+
'daemon': {
|
84
|
+
'status': job.daemon.status,
|
85
|
+
'pid': job.daemon.pid,
|
86
|
+
'properties': job.daemon.properties,
|
87
|
+
},
|
88
|
+
}
|
89
|
+
|
90
|
+
|
91
|
+
@app.post(endpoints['jobs'] + '/{name}', tags=['Jobs'])
|
92
|
+
def create_job(
|
93
|
+
name: str,
|
94
|
+
sysargs: List[str],
|
95
|
+
curr_user=(
|
96
|
+
fastapi.Depends(manager) if not no_auth else None
|
97
|
+
),
|
98
|
+
) -> SuccessTuple:
|
99
|
+
"""
|
100
|
+
Create and start a new job.
|
101
|
+
"""
|
102
|
+
job = Job(name, sysargs)
|
103
|
+
if job.exists():
|
104
|
+
raise fastapi.HTTPException(
|
105
|
+
status_code=409,
|
106
|
+
detail=f"{job} already exists."
|
107
|
+
)
|
108
|
+
|
109
|
+
return job.start()
|
110
|
+
|
111
|
+
|
112
|
+
@app.delete(endpoints['jobs'] + '/{name}', tags=['Jobs'])
|
113
|
+
def delete_job(
|
114
|
+
name: str,
|
115
|
+
curr_user=(
|
116
|
+
fastapi.Depends(manager) if not no_auth else None
|
117
|
+
),
|
118
|
+
) -> SuccessTuple:
|
119
|
+
"""
|
120
|
+
Delete a job.
|
121
|
+
"""
|
122
|
+
job = Job(name)
|
123
|
+
return job.delete()
|
124
|
+
|
125
|
+
|
126
|
+
@app.get(endpoints['jobs'] + '/{name}/exists', tags=['Jobs'])
|
127
|
+
def get_job_exists(
|
128
|
+
name: str,
|
129
|
+
curr_user=(
|
130
|
+
fastapi.Depends(manager) if not no_auth else None
|
131
|
+
),
|
132
|
+
) -> bool:
|
133
|
+
"""
|
134
|
+
Return whether a job exists.
|
135
|
+
"""
|
136
|
+
job = Job(name)
|
137
|
+
return job.exists()
|
138
|
+
|
139
|
+
|
140
|
+
@app.get(endpoints['logs'] + '/{name}', tags=['Jobs'])
|
141
|
+
def get_logs(
|
142
|
+
name: str,
|
143
|
+
curr_user=(
|
144
|
+
fastapi.Depends(manager) if not no_auth else None
|
145
|
+
),
|
146
|
+
) -> Union[str, None]:
|
147
|
+
"""
|
148
|
+
Return a job's log text.
|
149
|
+
To stream log text, connect to the WebSocket endpoint `/logs/{name}/ws`.
|
150
|
+
"""
|
151
|
+
job = Job(name)
|
152
|
+
if not job.exists():
|
153
|
+
raise fastapi.HTTPException(
|
154
|
+
status_code=404,
|
155
|
+
detail=f"{job} does not exist.",
|
156
|
+
)
|
157
|
+
|
158
|
+
return job.get_logs()
|
159
|
+
|
160
|
+
|
161
|
+
@app.post(endpoints['jobs'] + '/{name}/start', tags=['Jobs'])
|
162
|
+
def start_job(
|
163
|
+
name: str,
|
164
|
+
curr_user=(
|
165
|
+
fastapi.Depends(manager) if not no_auth else None
|
166
|
+
),
|
167
|
+
) -> SuccessTuple:
|
168
|
+
"""
|
169
|
+
Start a job if stopped.
|
170
|
+
"""
|
171
|
+
job = Job(name)
|
172
|
+
if not job.exists():
|
173
|
+
raise fastapi.HTTPException(
|
174
|
+
status_code=404,
|
175
|
+
detail=f"{job} does not exist."
|
176
|
+
)
|
177
|
+
return job.start()
|
178
|
+
|
179
|
+
|
180
|
+
@app.post(endpoints['jobs'] + '/{name}/stop', tags=['Jobs'])
|
181
|
+
def stop_job(
|
182
|
+
name: str,
|
183
|
+
curr_user=(
|
184
|
+
fastapi.Depends(manager) if not no_auth else None
|
185
|
+
),
|
186
|
+
) -> SuccessTuple:
|
187
|
+
"""
|
188
|
+
Stop a job if running.
|
189
|
+
"""
|
190
|
+
job = Job(name)
|
191
|
+
if not job.exists():
|
192
|
+
raise fastapi.HTTPException(
|
193
|
+
status_code=404,
|
194
|
+
detail=f"{job} does not exist."
|
195
|
+
)
|
196
|
+
return job.stop()
|
197
|
+
|
198
|
+
|
199
|
+
@app.post(endpoints['jobs'] + '/{name}/pause', tags=['Jobs'])
|
200
|
+
def pause_job(
|
201
|
+
name: str,
|
202
|
+
curr_user=(
|
203
|
+
fastapi.Depends(manager) if not no_auth else None
|
204
|
+
),
|
205
|
+
) -> SuccessTuple:
|
206
|
+
"""
|
207
|
+
Pause a job if running.
|
208
|
+
"""
|
209
|
+
job = Job(name)
|
210
|
+
if not job.exists():
|
211
|
+
raise fastapi.HTTPException(
|
212
|
+
status_code=404,
|
213
|
+
detail=f"{job} does not exist."
|
214
|
+
)
|
215
|
+
return job.pause()
|
216
|
+
|
217
|
+
|
218
|
+
@app.get(endpoints['jobs'] + '/{name}/stop_time', tags=['Jobs'])
|
219
|
+
def get_stop_time(
|
220
|
+
name: str,
|
221
|
+
curr_user=(
|
222
|
+
fastapi.Depends(manager) if not no_auth else None
|
223
|
+
),
|
224
|
+
) -> Union[datetime, None]:
|
225
|
+
"""
|
226
|
+
Get the timestamp when the job was manually stopped.
|
227
|
+
"""
|
228
|
+
job = Job(name)
|
229
|
+
return job.stop_time
|
230
|
+
|
231
|
+
|
232
|
+
@app.get(endpoints['jobs'] + '/{name}/is_blocking_on_stdin', tags=['Jobs'])
|
233
|
+
def get_is_blocking_on_stdin(
|
234
|
+
name: str,
|
235
|
+
curr_user=(
|
236
|
+
fastapi.Depends(manager) if not no_auth else None
|
237
|
+
),
|
238
|
+
) -> bool:
|
239
|
+
"""
|
240
|
+
Return whether a job is blocking on stdin.
|
241
|
+
"""
|
242
|
+
job = Job(name)
|
243
|
+
return job.is_blocking_on_stdin()
|
244
|
+
|
245
|
+
|
246
|
+
_job_clients = defaultdict(lambda: [])
|
247
|
+
_job_stop_events = defaultdict(lambda: asyncio.Event())
|
248
|
+
async def notify_clients(name: str, content: str):
|
249
|
+
"""
|
250
|
+
Write the given content to all connected clients.
|
251
|
+
"""
|
252
|
+
if not _job_clients[name]:
|
253
|
+
_job_stop_events[name].set()
|
254
|
+
|
255
|
+
async def _notify_client(client):
|
256
|
+
try:
|
257
|
+
await client.send_text(content)
|
258
|
+
except WebSocketDisconnect:
|
259
|
+
if client in _job_clients[name]:
|
260
|
+
_job_clients[name].remove(client)
|
261
|
+
except Exception:
|
262
|
+
pass
|
263
|
+
|
264
|
+
notify_tasks = [
|
265
|
+
asyncio.create_task(_notify_client(client))
|
266
|
+
for client in _job_clients[name]
|
267
|
+
]
|
268
|
+
await asyncio.wait(notify_tasks)
|
269
|
+
|
270
|
+
|
271
|
+
async def get_input_from_clients(name):
|
272
|
+
"""
|
273
|
+
When a job is blocking on input, return input from the first client which provides it.
|
274
|
+
"""
|
275
|
+
print('GET INPUT FROM CLIENTS')
|
276
|
+
if not _job_clients[name]:
|
277
|
+
print('NO CLIENTS')
|
278
|
+
return ''
|
279
|
+
|
280
|
+
async def _read_client(client):
|
281
|
+
try:
|
282
|
+
await client.send_text(JOBS_STDIN_MESSAGE)
|
283
|
+
data = await client.receive_text()
|
284
|
+
except WebSocketDisconnect:
|
285
|
+
if client in _job_clients[name]:
|
286
|
+
_job_clients[name].remove(client)
|
287
|
+
except Exception:
|
288
|
+
pass
|
289
|
+
return data
|
290
|
+
|
291
|
+
read_tasks = [
|
292
|
+
asyncio.create_task(_read_client(client))
|
293
|
+
for client in _job_clients[name]
|
294
|
+
]
|
295
|
+
done, pending = await asyncio.wait(read_tasks, return_when=asyncio.FIRST_COMPLETED)
|
296
|
+
for task in pending:
|
297
|
+
task.cancel()
|
298
|
+
for task in done:
|
299
|
+
return task.result()
|
300
|
+
|
301
|
+
|
302
|
+
@app.websocket(endpoints['logs'] + '/{name}/ws')
|
303
|
+
async def logs_websocket(name: str, websocket: WebSocket):
|
304
|
+
"""
|
305
|
+
Stream logs from a job over a websocket.
|
306
|
+
"""
|
307
|
+
await websocket.accept()
|
308
|
+
job = Job(name)
|
309
|
+
_job_clients[name].append(websocket)
|
310
|
+
|
311
|
+
async def monitor_logs():
|
312
|
+
await job.monitor_logs_async(
|
313
|
+
partial(notify_clients, name),
|
314
|
+
input_callback_function=partial(get_input_from_clients, name),
|
315
|
+
stop_event=_job_stop_events[name],
|
316
|
+
)
|
317
|
+
|
318
|
+
try:
|
319
|
+
token = await websocket.receive_text()
|
320
|
+
user = await manager.get_current_user(token) if not no_auth else None
|
321
|
+
if user is None and not no_auth:
|
322
|
+
raise fastapi.HTTPException(
|
323
|
+
status_code=401,
|
324
|
+
detail="Invalid credentials.",
|
325
|
+
)
|
326
|
+
monitor_task = asyncio.create_task(monitor_logs())
|
327
|
+
await monitor_task
|
328
|
+
except fastapi.HTTPException:
|
329
|
+
await websocket.send_text("Invalid credentials.")
|
330
|
+
await websocket.close()
|
331
|
+
except WebSocketDisconnect:
|
332
|
+
pass
|
333
|
+
except asyncio.CancelledError:
|
334
|
+
pass
|
335
|
+
except Exception:
|
336
|
+
warn(f"Error in logs websocket:\n{traceback.format_exc()}")
|
337
|
+
finally:
|
338
|
+
if websocket in _job_clients[name]:
|
339
|
+
_job_clients[name].remove(websocket)
|
340
|
+
_job_stop_events[name].clear()
|
meerschaum/api/routes/_pipes.py
CHANGED
@@ -201,13 +201,13 @@ async def fetch_pipes_keys(
|
|
201
201
|
|
202
202
|
@app.get(pipes_endpoint, tags=['Pipes'])
|
203
203
|
async def get_pipes(
|
204
|
-
connector_keys
|
205
|
-
metric_keys
|
206
|
-
location_keys
|
207
|
-
curr_user
|
204
|
+
connector_keys: str = "",
|
205
|
+
metric_keys: str = "",
|
206
|
+
location_keys: str = "",
|
207
|
+
curr_user=(
|
208
208
|
fastapi.Depends(manager) if not no_auth else None
|
209
209
|
),
|
210
|
-
debug
|
210
|
+
debug: bool = False,
|
211
211
|
) -> Dict[str, Any]:
|
212
212
|
"""
|
213
213
|
Get all registered Pipes with metadata, excluding parameters.
|
meerschaum/config/_default.py
CHANGED
meerschaum/config/_paths.py
CHANGED
@@ -180,6 +180,7 @@ paths = {
|
|
180
180
|
'DAEMON_RESOURCES_PATH' : ('{ROOT_DIR_PATH}', 'jobs'),
|
181
181
|
'LOGS_RESOURCES_PATH' : ('{ROOT_DIR_PATH}', 'logs'),
|
182
182
|
'DAEMON_ERROR_LOG_PATH' : ('{ROOT_DIR_PATH}', 'daemon_errors.log'),
|
183
|
+
'CHECK_JOBS_LOCK_PATH' : ('{INTERNAL_RESOURCES_PATH}', 'check-jobs.lock'),
|
183
184
|
}
|
184
185
|
|
185
186
|
def set_root(root: Union[Path, str]):
|
meerschaum/config/_shell.py
CHANGED
@@ -76,6 +76,11 @@ default_shell_config = {
|
|
76
76
|
'magenta',
|
77
77
|
],
|
78
78
|
},
|
79
|
+
'executor' : {
|
80
|
+
'rich' : {
|
81
|
+
'style' : 'gold1',
|
82
|
+
},
|
83
|
+
},
|
79
84
|
'username' : {
|
80
85
|
'rich' : {
|
81
86
|
'style' : 'white',
|
@@ -112,8 +117,8 @@ default_shell_config = {
|
|
112
117
|
'ascii' : {
|
113
118
|
'intro' : r""" ___ ___ __ __ __
|
114
119
|
|\/| |__ |__ |__) /__` / ` |__| /\ | | |\/|
|
115
|
-
| | |___ |___ | \ .__/ \__, | | /~~\ \__/ |
|
116
|
-
'prompt' : '\n [ {username}@{instance} ] > ',
|
120
|
+
| | |___ |___ | \ .__/ \__, | | /~~\ \__/ | |""" + '\n',
|
121
|
+
'prompt' : '\n [ {username}@{instance} | {executor_keys} ] > ',
|
117
122
|
'ruler' : '-',
|
118
123
|
'close_message' : 'Thank you for using Meerschaum!',
|
119
124
|
'doc_header' : 'Meerschaum actions (`help <action>` for usage):',
|
@@ -124,7 +129,7 @@ default_shell_config = {
|
|
124
129
|
'intro' : """
|
125
130
|
█▄ ▄█ ██▀ ██▀ █▀▄ ▄▀▀ ▄▀▀ █▄█ ▄▀▄ █ █ █▄ ▄█
|
126
131
|
█ ▀ █ █▄▄ █▄▄ █▀▄ ▄██ ▀▄▄ █ █ █▀█ ▀▄█ █ ▀ █\n""",
|
127
|
-
'prompt' : '\n [ {username}@{instance} ] ➤ ',
|
132
|
+
'prompt' : '\n [ {username}@{instance} | {executor_keys} ] ➤ ',
|
128
133
|
'ruler' : '─',
|
129
134
|
'close_message' : ' MRSM{formatting:emoji:hand} Thank you for using Meerschaum! ',
|
130
135
|
'doc_header' : 'Meerschaum actions (`help <action>` for usage):',
|
meerschaum/config/_version.py
CHANGED
@@ -23,6 +23,8 @@ STATIC_CONFIG: Dict[str, Any] = {
|
|
23
23
|
'pipes': '/pipes',
|
24
24
|
'metadata': '/metadata',
|
25
25
|
'actions': '/actions',
|
26
|
+
'jobs': '/jobs',
|
27
|
+
'logs': '/logs',
|
26
28
|
'users': '/users',
|
27
29
|
'login': '/login',
|
28
30
|
'connectors': '/connectors',
|
@@ -40,6 +42,9 @@ STATIC_CONFIG: Dict[str, Any] = {
|
|
40
42
|
},
|
41
43
|
'webterm_job_name': '_webterm',
|
42
44
|
'default_timeout': 600,
|
45
|
+
'jobs': {
|
46
|
+
'stdin_message': 'MRSM_STDIN'
|
47
|
+
},
|
43
48
|
},
|
44
49
|
'sql': {
|
45
50
|
'internal_schema': '_mrsm_internal',
|
@@ -129,6 +134,9 @@ STATIC_CONFIG: Dict[str, Any] = {
|
|
129
134
|
},
|
130
135
|
'exists_timeout_seconds': 5.0,
|
131
136
|
},
|
137
|
+
'jobs': {
|
138
|
+
'check_restart_seconds': 1.0,
|
139
|
+
},
|
132
140
|
'setup': {
|
133
141
|
'name': 'meerschaum',
|
134
142
|
'formal_name': 'Meerschaum',
|
@@ -70,12 +70,12 @@ _loaded_plugin_connectors: bool = False
|
|
70
70
|
|
71
71
|
|
72
72
|
def get_connector(
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
73
|
+
type: str = None,
|
74
|
+
label: str = None,
|
75
|
+
refresh: bool = False,
|
76
|
+
debug: bool = False,
|
77
|
+
**kw: Any
|
78
|
+
) -> Connector:
|
79
79
|
"""
|
80
80
|
Return existing connector or create new connection and store for reuse.
|
81
81
|
|
@@ -274,9 +274,7 @@ def is_connected(keys: str, **kw) -> bool:
|
|
274
274
|
return False
|
275
275
|
|
276
276
|
|
277
|
-
def make_connector(
|
278
|
-
cls,
|
279
|
-
):
|
277
|
+
def make_connector(cls):
|
280
278
|
"""
|
281
279
|
Register a class as a `Connector`.
|
282
280
|
The `type` will be the lower case of the class name, without the suffix `connector`.
|
@@ -338,8 +336,8 @@ def load_plugin_connectors():
|
|
338
336
|
|
339
337
|
|
340
338
|
def get_connector_plugin(
|
341
|
-
|
342
|
-
|
339
|
+
connector: Connector,
|
340
|
+
) -> Union[str, None, mrsm.Plugin]:
|
343
341
|
"""
|
344
342
|
Determine the plugin for a connector.
|
345
343
|
This is useful for handling virtual environments for custom instance connectors.
|