meerschaum 2.2.6__py3-none-any.whl → 2.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. meerschaum/__init__.py +6 -1
  2. meerschaum/__main__.py +9 -9
  3. meerschaum/_internal/arguments/__init__.py +1 -1
  4. meerschaum/_internal/arguments/_parse_arguments.py +72 -6
  5. meerschaum/_internal/arguments/_parser.py +45 -15
  6. meerschaum/_internal/docs/index.py +265 -8
  7. meerschaum/_internal/entry.py +167 -37
  8. meerschaum/_internal/shell/Shell.py +290 -99
  9. meerschaum/_internal/shell/updates.py +175 -0
  10. meerschaum/actions/__init__.py +29 -17
  11. meerschaum/actions/api.py +12 -12
  12. meerschaum/actions/attach.py +113 -0
  13. meerschaum/actions/copy.py +68 -41
  14. meerschaum/actions/delete.py +112 -50
  15. meerschaum/actions/edit.py +3 -3
  16. meerschaum/actions/install.py +40 -32
  17. meerschaum/actions/pause.py +44 -27
  18. meerschaum/actions/register.py +19 -5
  19. meerschaum/actions/restart.py +107 -0
  20. meerschaum/actions/show.py +130 -159
  21. meerschaum/actions/start.py +161 -100
  22. meerschaum/actions/stop.py +78 -42
  23. meerschaum/actions/sync.py +3 -3
  24. meerschaum/actions/upgrade.py +28 -36
  25. meerschaum/api/_events.py +25 -1
  26. meerschaum/api/_oauth2.py +2 -0
  27. meerschaum/api/_websockets.py +2 -2
  28. meerschaum/api/dash/callbacks/jobs.py +36 -44
  29. meerschaum/api/dash/jobs.py +89 -78
  30. meerschaum/api/routes/__init__.py +1 -0
  31. meerschaum/api/routes/_actions.py +148 -17
  32. meerschaum/api/routes/_jobs.py +407 -0
  33. meerschaum/api/routes/_pipes.py +25 -25
  34. meerschaum/config/_default.py +1 -0
  35. meerschaum/config/_formatting.py +1 -0
  36. meerschaum/config/_jobs.py +1 -1
  37. meerschaum/config/_paths.py +11 -0
  38. meerschaum/config/_shell.py +84 -67
  39. meerschaum/config/_version.py +1 -1
  40. meerschaum/config/static/__init__.py +18 -0
  41. meerschaum/connectors/Connector.py +13 -7
  42. meerschaum/connectors/__init__.py +28 -15
  43. meerschaum/connectors/api/APIConnector.py +27 -1
  44. meerschaum/connectors/api/_actions.py +71 -6
  45. meerschaum/connectors/api/_jobs.py +368 -0
  46. meerschaum/connectors/api/_misc.py +1 -1
  47. meerschaum/connectors/api/_pipes.py +85 -84
  48. meerschaum/connectors/api/_request.py +13 -9
  49. meerschaum/connectors/parse.py +27 -15
  50. meerschaum/core/Pipe/_bootstrap.py +16 -8
  51. meerschaum/core/Pipe/_sync.py +3 -0
  52. meerschaum/jobs/_Executor.py +69 -0
  53. meerschaum/jobs/_Job.py +899 -0
  54. meerschaum/jobs/__init__.py +396 -0
  55. meerschaum/jobs/systemd.py +694 -0
  56. meerschaum/plugins/__init__.py +97 -12
  57. meerschaum/utils/daemon/Daemon.py +352 -147
  58. meerschaum/utils/daemon/FileDescriptorInterceptor.py +19 -10
  59. meerschaum/utils/daemon/RotatingFile.py +22 -8
  60. meerschaum/utils/daemon/StdinFile.py +121 -0
  61. meerschaum/utils/daemon/__init__.py +42 -27
  62. meerschaum/utils/daemon/_names.py +15 -13
  63. meerschaum/utils/formatting/__init__.py +83 -37
  64. meerschaum/utils/formatting/_jobs.py +146 -55
  65. meerschaum/utils/formatting/_shell.py +6 -0
  66. meerschaum/utils/misc.py +41 -22
  67. meerschaum/utils/packages/__init__.py +21 -15
  68. meerschaum/utils/packages/_packages.py +9 -6
  69. meerschaum/utils/process.py +9 -9
  70. meerschaum/utils/prompt.py +20 -7
  71. meerschaum/utils/schedule.py +21 -15
  72. meerschaum/utils/venv/__init__.py +2 -2
  73. {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/METADATA +22 -25
  74. {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/RECORD +80 -70
  75. {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/WHEEL +1 -1
  76. {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/LICENSE +0 -0
  77. {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/NOTICE +0 -0
  78. {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/entry_points.txt +0 -0
  79. {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/top_level.txt +0 -0
  80. {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/zip-safe +0 -0
@@ -7,46 +7,176 @@ Execute Meerschaum Actions via the API
7
7
  """
8
8
 
9
9
  from __future__ import annotations
10
- from meerschaum.utils.typing import SuccessTuple, Union
10
+ import asyncio
11
+ import traceback
12
+ import shlex
13
+ from functools import partial
14
+ from datetime import datetime, timezone
11
15
 
16
+ from fastapi import WebSocket, WebSocketDisconnect
17
+
18
+ from meerschaum.utils.misc import generate_password
19
+ from meerschaum.jobs import Job
20
+ from meerschaum.utils.warnings import warn
21
+ from meerschaum.utils.typing import SuccessTuple, Union, List, Dict, Any
12
22
  from meerschaum.api import (
13
23
  fastapi, app, endpoints, get_api_connector, debug, manager, private, no_auth
14
24
  )
15
25
  from meerschaum.actions import actions
16
26
  import meerschaum.core
27
+ from meerschaum.config import get_config
28
+ from meerschaum._internal.arguments._parse_arguments import parse_dict_to_sysargs, parse_arguments
29
+ from meerschaum.api.routes._jobs import clean_sysargs
30
+
17
31
  actions_endpoint = endpoints['actions']
18
32
 
33
+ def is_user_allowed_to_execute(user) -> SuccessTuple:
34
+ if user is None:
35
+ return False, "Could not load user."
36
+
37
+ if user.type == 'admin':
38
+ return True, "Success"
39
+
40
+ allow_non_admin = get_config(
41
+ 'system', 'api', 'permissions', 'actions', 'non_admin', patch=True
42
+ )
43
+ if not allow_non_admin:
44
+ return False, (
45
+ "The administrator for this server has not allowed users to perform actions.\n\n"
46
+ + "Please contact the system administrator, or if you are running this server, "
47
+ + "open the configuration file with `edit config system` "
48
+ + "and search for 'permissions'. "
49
+ + "\nUnder the keys 'api:permissions:actions', "
50
+ + "you can allow non-admin users to perform actions."
51
+ )
52
+
53
+ return True, "Success"
54
+
55
+
19
56
  @app.get(actions_endpoint, tags=['Actions'])
20
57
  def get_actions(
21
- curr_user = (
22
- fastapi.Depends(manager) if private else None
23
- ),
24
- ) -> list:
58
+ curr_user = (
59
+ fastapi.Depends(manager) if private else None
60
+ ),
61
+ ) -> List[str]:
62
+ """
63
+ Return a list of the available actions.
64
+ """
25
65
  return list(actions)
26
66
 
67
+
68
+ async def notify_client(client, content: str):
69
+ """
70
+ Send a line of text to a client.
71
+ """
72
+ try:
73
+ await client.send_text(content)
74
+ except WebSocketDisconnect:
75
+ pass
76
+
77
+ _temp_jobs = {}
78
+ @app.websocket(actions_endpoint + '/ws')
79
+ async def do_action_websocket(websocket: WebSocket):
80
+ """
81
+ Execute an action and stream the output to the client.
82
+ """
83
+ await websocket.accept()
84
+
85
+ stop_event = asyncio.Event()
86
+
87
+ async def monitor_logs(job):
88
+ success, msg = job.start()
89
+ await job.monitor_logs_async(
90
+ partial(notify_client, websocket),
91
+ stop_event=stop_event,
92
+ stop_on_exit=True,
93
+ )
94
+
95
+ job = None
96
+ job_name = '.' + generate_password(12)
97
+ try:
98
+ token = await websocket.receive_text()
99
+ user = await manager.get_current_user(token) if not no_auth else None
100
+ if user is None and not no_auth:
101
+ raise fastapi.HTTPException(
102
+ status_code=401,
103
+ detail="Invalid credentials.",
104
+ )
105
+
106
+ auth_success, auth_msg = (
107
+ is_user_allowed_to_execute(user)
108
+ if not no_auth
109
+ else (True, "Success")
110
+ )
111
+ auth_payload = {
112
+ 'is_authenticated': auth_success,
113
+ 'timestamp': datetime.now(timezone.utc).isoformat(),
114
+ }
115
+ await websocket.send_json(auth_payload)
116
+ if not auth_success:
117
+ await websocket.close()
118
+
119
+ sysargs = clean_sysargs(await websocket.receive_json())
120
+ # kwargs = parse_arguments(sysargs)
121
+ # _ = kwargs.pop('executor_keys', None)
122
+ # _ = kwargs.pop('shell', None)
123
+ # sysargs = parse_dict_to_sysargs(kwargs)
124
+
125
+ job = Job(
126
+ job_name,
127
+ sysargs,
128
+ executor_keys='local',
129
+ _properties={
130
+ 'logs': {
131
+ 'write_timestamps': False,
132
+ },
133
+ },
134
+ )
135
+ _temp_jobs[job_name] = job
136
+ monitor_task = asyncio.create_task(monitor_logs(job))
137
+ await monitor_task
138
+ try:
139
+ await websocket.close()
140
+ except RuntimeError:
141
+ pass
142
+ except fastapi.HTTPException:
143
+ await websocket.send_text("Invalid credentials.")
144
+ await websocket.close()
145
+ except WebSocketDisconnect:
146
+ pass
147
+ except asyncio.CancelledError:
148
+ pass
149
+ except Exception:
150
+ warn(f"Error in logs websocket:\n{traceback.format_exc()}")
151
+ finally:
152
+ if job is not None:
153
+ job.delete()
154
+ _ = _temp_jobs.pop(job_name, None)
155
+ stop_event.set()
156
+
157
+
27
158
  @app.post(actions_endpoint + "/{action}", tags=['Actions'])
28
- def do_action(
29
- action: str,
30
- keywords: dict = fastapi.Body(...),
31
- curr_user = (
32
- fastapi.Depends(manager) if not no_auth else None
33
- ),
34
- ) -> SuccessTuple:
159
+ def do_action_legacy(
160
+ action: str,
161
+ keywords: Dict[str, Any] = fastapi.Body(...),
162
+ curr_user = (
163
+ fastapi.Depends(manager) if not no_auth else None
164
+ ),
165
+ ) -> SuccessTuple:
35
166
  """
36
- Perform a Meerschaum action (if permissions allow it).
167
+ Perform a Meerschaum action (if permissions allow).
37
168
 
38
169
  Parameters
39
170
  ----------
40
- action: str :
171
+ action: str
41
172
  The action to perform.
42
173
 
43
- keywords: dict :
174
+ keywords: Dict[str, Any]
44
175
  The keywords dictionary to pass to the action.
45
176
 
46
177
  Returns
47
178
  -------
48
- A `SuccessTuple` of success, message.
49
-
179
+ A `SuccessTuple`.
50
180
  """
51
181
  if curr_user is not None and curr_user.type != 'admin':
52
182
  from meerschaum.config import get_config
@@ -65,6 +195,7 @@ def do_action(
65
195
 
66
196
  if action not in actions:
67
197
  return False, f"Invalid action '{action}'."
198
+
68
199
  keywords['mrsm_instance'] = keywords.get('mrsm_instance', str(get_api_connector()))
69
200
  _debug = keywords.get('debug', debug)
70
201
  keywords.pop('debug', None)
@@ -0,0 +1,407 @@
1
+ #! /usr/bin/env python3
2
+ # -*- coding: utf-8 -*-
3
+ # vim:fenc=utf-8
4
+
5
+ """
6
+ Manage jobs via the Meerschaum API.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import os
12
+ import select
13
+ import asyncio
14
+ import traceback
15
+ from datetime import datetime
16
+ from collections import defaultdict
17
+ from functools import partial
18
+
19
+ from fastapi import WebSocket, WebSocketDisconnect
20
+
21
+ from meerschaum.utils.typing import Dict, Any, SuccessTuple, List, Optional, Union
22
+ from meerschaum.jobs import (
23
+ get_jobs as _get_jobs,
24
+ Job,
25
+ StopMonitoringLogs,
26
+ get_executor_keys_from_context,
27
+ )
28
+ from meerschaum.utils.warnings import warn
29
+
30
+ from meerschaum.api import (
31
+ fastapi,
32
+ app,
33
+ endpoints,
34
+ manager,
35
+ debug,
36
+ no_auth,
37
+ private,
38
+ )
39
+ from meerschaum.config.static import STATIC_CONFIG
40
+
41
+ JOBS_STDIN_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stdin_message']
42
+ JOBS_STOP_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stop_message']
43
+ EXECUTOR_KEYS: str = get_executor_keys_from_context()
44
+
45
+
46
+ @app.get(endpoints['jobs'], tags=['Jobs'])
47
+ def get_jobs(
48
+ curr_user=(
49
+ fastapi.Depends(manager) if not no_auth else None
50
+ ),
51
+ ) -> Dict[str, Dict[str, Any]]:
52
+ """
53
+ Return metadata about the current jobs.
54
+ """
55
+ jobs = _get_jobs(executor_keys=EXECUTOR_KEYS, combine_local_and_systemd=False)
56
+ return {
57
+ name: {
58
+ 'sysargs': job.sysargs,
59
+ 'result': job.result,
60
+ 'restart': job.restart,
61
+ 'status': job.status,
62
+ 'daemon': {
63
+ 'status': job.daemon.status if job.executor_keys is None else job.status,
64
+ 'pid': job.pid,
65
+ 'properties': (
66
+ job.daemon.properties
67
+ if job.executor is None
68
+ else job.executor.get_job_properties(name)
69
+ ),
70
+ },
71
+ }
72
+ for name, job in jobs.items()
73
+ }
74
+
75
+
76
+ @app.get(endpoints['jobs'] + '/{name}', tags=['Jobs'])
77
+ def get_job(
78
+ name: str,
79
+ curr_user=(
80
+ fastapi.Depends(manager) if not no_auth else None
81
+ ),
82
+ ) -> Dict[str, Any]:
83
+ """
84
+ Return metadata for a single job.
85
+ """
86
+ job = Job(name, executor_keys=EXECUTOR_KEYS)
87
+ if not job.exists():
88
+ raise fastapi.HTTPException(
89
+ status_code=404,
90
+ detail=f"{job} doesn't exist."
91
+ )
92
+
93
+ return {
94
+ 'sysargs': job.sysargs,
95
+ 'result': job.result,
96
+ 'restart': job.restart,
97
+ 'status': job.status,
98
+ 'daemon': {
99
+ 'status': job.daemon.status if job.executor_keys is None else job.status,
100
+ 'pid': job.pid,
101
+ 'properties': (
102
+ job.daemon.properties
103
+ if job.executor is None
104
+ else job.executor.get_job_properties(job.name)
105
+ ),
106
+ },
107
+ }
108
+
109
+
110
+ def clean_sysargs(sysargs: List[str]) -> List[str]:
111
+ """
112
+ Remove the executor flag or leading `api {label}` action.
113
+ """
114
+ clean_sysargs = []
115
+ executor_flag = False
116
+ for arg in sysargs:
117
+ if arg in ('-e', '--executor', 'api'):
118
+ executor_flag = True
119
+ continue
120
+ if executor_flag:
121
+ executor_flag = False
122
+ continue
123
+
124
+ clean_sysargs.append(arg)
125
+ return clean_sysargs
126
+
127
+
128
+ @app.post(endpoints['jobs'] + '/{name}', tags=['Jobs'])
129
+ def create_job(
130
+ name: str,
131
+ sysargs: List[str],
132
+ curr_user=(
133
+ fastapi.Depends(manager) if not no_auth else None
134
+ ),
135
+ ) -> SuccessTuple:
136
+ """
137
+ Create and start a new job.
138
+ """
139
+ job = Job(name, clean_sysargs(sysargs), executor_keys=EXECUTOR_KEYS)
140
+ if job.exists():
141
+ raise fastapi.HTTPException(
142
+ status_code=409,
143
+ detail=f"{job} already exists."
144
+ )
145
+
146
+ return job.start()
147
+
148
+
149
+ @app.delete(endpoints['jobs'] + '/{name}', tags=['Jobs'])
150
+ def delete_job(
151
+ name: str,
152
+ curr_user=(
153
+ fastapi.Depends(manager) if not no_auth else None
154
+ ),
155
+ ) -> SuccessTuple:
156
+ """
157
+ Delete a job.
158
+ """
159
+ job = Job(name, executor_keys=EXECUTOR_KEYS)
160
+ return job.delete()
161
+
162
+
163
+ @app.get(endpoints['jobs'] + '/{name}/exists', tags=['Jobs'])
164
+ def get_job_exists(
165
+ name: str,
166
+ curr_user=(
167
+ fastapi.Depends(manager) if not no_auth else None
168
+ ),
169
+ ) -> bool:
170
+ """
171
+ Return whether a job exists.
172
+ """
173
+ job = Job(name, executor_keys=EXECUTOR_KEYS)
174
+ return job.exists()
175
+
176
+
177
+ @app.get(endpoints['logs'] + '/{name}', tags=['Jobs'])
178
+ def get_logs(
179
+ name: str,
180
+ curr_user=(
181
+ fastapi.Depends(manager) if not no_auth else None
182
+ ),
183
+ ) -> Union[str, None]:
184
+ """
185
+ Return a job's log text.
186
+ To stream log text, connect to the WebSocket endpoint `/logs/{name}/ws`.
187
+ """
188
+ job = Job(name, executor_keys=EXECUTOR_KEYS)
189
+ if not job.exists():
190
+ raise fastapi.HTTPException(
191
+ status_code=404,
192
+ detail=f"{job} does not exist.",
193
+ )
194
+
195
+ return job.get_logs()
196
+
197
+
198
+ @app.post(endpoints['jobs'] + '/{name}/start', tags=['Jobs'])
199
+ def start_job(
200
+ name: str,
201
+ curr_user=(
202
+ fastapi.Depends(manager) if not no_auth else None
203
+ ),
204
+ ) -> SuccessTuple:
205
+ """
206
+ Start a job if stopped.
207
+ """
208
+ job = Job(name, executor_keys=EXECUTOR_KEYS)
209
+ if not job.exists():
210
+ raise fastapi.HTTPException(
211
+ status_code=404,
212
+ detail=f"{job} does not exist."
213
+ )
214
+ return job.start()
215
+
216
+
217
+ @app.post(endpoints['jobs'] + '/{name}/stop', tags=['Jobs'])
218
+ def stop_job(
219
+ name: str,
220
+ curr_user=(
221
+ fastapi.Depends(manager) if not no_auth else None
222
+ ),
223
+ ) -> SuccessTuple:
224
+ """
225
+ Stop a job if running.
226
+ """
227
+ job = Job(name, executor_keys=EXECUTOR_KEYS)
228
+ if not job.exists():
229
+ raise fastapi.HTTPException(
230
+ status_code=404,
231
+ detail=f"{job} does not exist."
232
+ )
233
+ return job.stop()
234
+
235
+
236
+ @app.post(endpoints['jobs'] + '/{name}/pause', tags=['Jobs'])
237
+ def pause_job(
238
+ name: str,
239
+ curr_user=(
240
+ fastapi.Depends(manager) if not no_auth else None
241
+ ),
242
+ ) -> SuccessTuple:
243
+ """
244
+ Pause a job if running.
245
+ """
246
+ job = Job(name, executor_keys=EXECUTOR_KEYS)
247
+ if not job.exists():
248
+ raise fastapi.HTTPException(
249
+ status_code=404,
250
+ detail=f"{job} does not exist."
251
+ )
252
+ return job.pause()
253
+
254
+
255
+ @app.get(endpoints['jobs'] + '/{name}/stop_time', tags=['Jobs'])
256
+ def get_stop_time(
257
+ name: str,
258
+ curr_user=(
259
+ fastapi.Depends(manager) if not no_auth else None
260
+ ),
261
+ ) -> Union[datetime, None]:
262
+ """
263
+ Get the timestamp when the job was manually stopped.
264
+ """
265
+ job = Job(name, executor_keys=EXECUTOR_KEYS)
266
+ return job.stop_time
267
+
268
+
269
+ @app.get(endpoints['jobs'] + '/{name}/is_blocking_on_stdin', tags=['Jobs'])
270
+ def get_is_blocking_on_stdin(
271
+ name: str,
272
+ curr_user=(
273
+ fastapi.Depends(manager) if not no_auth else None
274
+ ),
275
+ ) -> bool:
276
+ """
277
+ Return whether a job is blocking on stdin.
278
+ """
279
+ job = Job(name, executor_keys=EXECUTOR_KEYS)
280
+ return job.is_blocking_on_stdin()
281
+
282
+
283
+ _job_clients = defaultdict(lambda: [])
284
+ _job_stop_events = defaultdict(lambda: asyncio.Event())
285
+ async def notify_clients(name: str, websocket: WebSocket, content: str):
286
+ """
287
+ Write the given content to all connected clients.
288
+ """
289
+ async def _notify_client(client):
290
+ try:
291
+ await client.send_text(content)
292
+ except WebSocketDisconnect:
293
+ if client in _job_clients[name]:
294
+ _job_clients[name].remove(client)
295
+ except Exception:
296
+ pass
297
+
298
+ await _notify_client(websocket)
299
+
300
+
301
+ async def get_input_from_clients(name: str, websocket: WebSocket) -> str:
302
+ """
303
+ When a job is blocking on input, return input from the first client which provides it.
304
+ """
305
+ if not _job_clients[name]:
306
+ return ''
307
+
308
+ async def _read_client(client):
309
+ try:
310
+ await client.send_text(JOBS_STDIN_MESSAGE)
311
+ data = await client.receive_text()
312
+ except WebSocketDisconnect:
313
+ if client in _job_clients[name]:
314
+ _job_clients[name].remove(client)
315
+ except Exception:
316
+ pass
317
+ return data
318
+
319
+ read_tasks = [
320
+ asyncio.create_task(_read_client(client))
321
+ for client in _job_clients[name]
322
+ ]
323
+ done, pending = await asyncio.wait(read_tasks, return_when=asyncio.FIRST_COMPLETED)
324
+ for task in pending:
325
+ task.cancel()
326
+ for task in done:
327
+ return task.result()
328
+
329
+
330
+ async def send_stop_message(name: str, client: WebSocket, result: SuccessTuple):
331
+ """
332
+ Send a stop message to clients when the job stops.
333
+ """
334
+ try:
335
+ await client.send_text(JOBS_STOP_MESSAGE)
336
+ await client.send_json(result)
337
+ except WebSocketDisconnect:
338
+ _job_stop_events[name].set()
339
+ if client in _job_clients[name]:
340
+ _job_clients[name].remove(client)
341
+ except RuntimeError:
342
+ pass
343
+ except Exception:
344
+ warn(traceback.format_exc())
345
+
346
+
347
+ @app.websocket(endpoints['logs'] + '/{name}/ws')
348
+ async def logs_websocket(name: str, websocket: WebSocket):
349
+ """
350
+ Stream logs from a job over a websocket.
351
+ """
352
+ await websocket.accept()
353
+ job = Job(name, executor_keys=EXECUTOR_KEYS)
354
+ _job_clients[name].append(websocket)
355
+
356
+ async def monitor_logs():
357
+ try:
358
+ callback_function = partial(
359
+ notify_clients,
360
+ name,
361
+ websocket,
362
+ )
363
+ input_callback_function = partial(
364
+ get_input_from_clients,
365
+ name,
366
+ websocket,
367
+ )
368
+ stop_callback_function = partial(
369
+ send_stop_message,
370
+ name,
371
+ websocket,
372
+ )
373
+ await job.monitor_logs_async(
374
+ callback_function=callback_function,
375
+ input_callback_function=input_callback_function,
376
+ stop_callback_function=stop_callback_function,
377
+ stop_event=_job_stop_events[name],
378
+ stop_on_exit=True,
379
+ accept_input=True,
380
+ )
381
+ except Exception:
382
+ warn(traceback.format_exc())
383
+
384
+ try:
385
+ token = await websocket.receive_text()
386
+ user = await manager.get_current_user(token) if not no_auth else None
387
+ if user is None and not no_auth:
388
+ raise fastapi.HTTPException(
389
+ status_code=401,
390
+ detail="Invalid credentials.",
391
+ )
392
+ monitor_task = asyncio.create_task(monitor_logs())
393
+ await monitor_task
394
+ except fastapi.HTTPException:
395
+ await websocket.send_text("Invalid credentials.")
396
+ await websocket.close()
397
+ except WebSocketDisconnect:
398
+ _job_stop_events[name].set()
399
+ monitor_task.cancel()
400
+ except asyncio.CancelledError:
401
+ pass
402
+ except Exception:
403
+ warn(f"Error in logs websocket:\n{traceback.format_exc()}")
404
+ finally:
405
+ if websocket in _job_clients[name]:
406
+ _job_clients[name].remove(websocket)
407
+ _job_stop_events[name].clear()
@@ -44,14 +44,14 @@ pd = attempt_import('pandas')
44
44
 
45
45
  @app.post(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/register', tags=['Pipes'])
46
46
  def register_pipe(
47
- connector_keys: str,
48
- metric_key: str,
49
- location_key: str,
50
- parameters: dict,
51
- curr_user = (
52
- fastapi.Depends(manager) if not no_auth else None
53
- ),
54
- ):
47
+ connector_keys: str,
48
+ metric_key: str,
49
+ location_key: str,
50
+ parameters: dict,
51
+ curr_user = (
52
+ fastapi.Depends(manager) if not no_auth else None
53
+ ),
54
+ ):
55
55
  """
56
56
  Register a new pipe.
57
57
  """
@@ -201,13 +201,13 @@ async def fetch_pipes_keys(
201
201
 
202
202
  @app.get(pipes_endpoint, tags=['Pipes'])
203
203
  async def get_pipes(
204
- connector_keys : str = "",
205
- metric_keys : str = "",
206
- location_keys : str = "",
207
- curr_user = (
204
+ connector_keys: str = "",
205
+ metric_keys: str = "",
206
+ location_keys: str = "",
207
+ curr_user=(
208
208
  fastapi.Depends(manager) if not no_auth else None
209
209
  ),
210
- debug : bool = False
210
+ debug: bool = False,
211
211
  ) -> Dict[str, Any]:
212
212
  """
213
213
  Get all registered Pipes with metadata, excluding parameters.
@@ -376,18 +376,18 @@ def sync_pipe(
376
376
 
377
377
  @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/data', tags=['Pipes'])
378
378
  def get_pipe_data(
379
- connector_keys: str,
380
- metric_key: str,
381
- location_key: str,
382
- select_columns: Optional[str] = None,
383
- omit_columns: Optional[str] = None,
384
- begin: Union[str, int, None] = None,
385
- end: Union[str, int, None] = None,
386
- params: Optional[str] = None,
387
- curr_user = (
388
- fastapi.Depends(manager) if not no_auth else None
389
- ),
390
- ) -> str:
379
+ connector_keys: str,
380
+ metric_key: str,
381
+ location_key: str,
382
+ select_columns: Optional[str] = None,
383
+ omit_columns: Optional[str] = None,
384
+ begin: Union[str, int, None] = None,
385
+ end: Union[str, int, None] = None,
386
+ params: Optional[str] = None,
387
+ curr_user = (
388
+ fastapi.Depends(manager) if not no_auth else None
389
+ ),
390
+ ) -> str:
391
391
  """
392
392
  Get a pipe's data, applying any filtering.
393
393
 
@@ -16,6 +16,7 @@ default_meerschaum_config = {
16
16
  'api_instance': 'MRSM{meerschaum:instance}',
17
17
  'web_instance': 'MRSM{meerschaum:instance}',
18
18
  'default_repository': 'api:mrsm',
19
+ # 'default_executor': 'local',
19
20
  'connectors': {
20
21
  'sql': {
21
22
  'default': {},
@@ -36,6 +36,7 @@ default_formatting_config = {
36
36
  'paused' : '🟡',
37
37
  'stopped' : '🔴',
38
38
  'tag' : '🔖',
39
+ 'announcement' : '📢',
39
40
  },
40
41
  'pipes' : {
41
42
  'unicode' : {
@@ -7,7 +7,7 @@ Default configuration for jobs.
7
7
  """
8
8
 
9
9
  default_jobs_config = {
10
- 'timeout_seconds': 8,
10
+ 'timeout_seconds': 4,
11
11
  'check_timeout_interval_seconds': 0.1,
12
12
  'terminal': {
13
13
  'lines': 40,