meerschaum 2.3.0.dev1__py3-none-any.whl → 2.3.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/__init__.py +5 -2
- meerschaum/__main__.py +0 -5
- meerschaum/_internal/arguments/_parse_arguments.py +10 -3
- meerschaum/_internal/arguments/_parser.py +6 -2
- meerschaum/_internal/entry.py +36 -6
- meerschaum/_internal/shell/Shell.py +32 -20
- meerschaum/actions/__init__.py +8 -6
- meerschaum/actions/attach.py +31 -13
- meerschaum/actions/copy.py +68 -41
- meerschaum/actions/delete.py +64 -21
- meerschaum/actions/edit.py +3 -3
- meerschaum/actions/install.py +40 -32
- meerschaum/actions/pause.py +44 -27
- meerschaum/actions/restart.py +107 -0
- meerschaum/actions/show.py +8 -8
- meerschaum/actions/start.py +26 -41
- meerschaum/actions/stop.py +11 -4
- meerschaum/api/_events.py +10 -3
- meerschaum/api/dash/jobs.py +69 -70
- meerschaum/api/routes/_actions.py +8 -3
- meerschaum/api/routes/_jobs.py +86 -37
- meerschaum/config/_default.py +1 -1
- meerschaum/config/_paths.py +5 -0
- meerschaum/config/_shell.py +1 -1
- meerschaum/config/_version.py +1 -1
- meerschaum/config/static/__init__.py +6 -1
- meerschaum/connectors/Connector.py +13 -7
- meerschaum/connectors/__init__.py +21 -5
- meerschaum/connectors/api/APIConnector.py +3 -0
- meerschaum/connectors/api/_jobs.py +108 -11
- meerschaum/connectors/parse.py +10 -13
- meerschaum/core/Pipe/_bootstrap.py +16 -8
- meerschaum/jobs/_Executor.py +69 -0
- meerschaum/{utils/jobs → jobs}/_Job.py +206 -40
- meerschaum/jobs/_LocalExecutor.py +88 -0
- meerschaum/jobs/_SystemdExecutor.py +608 -0
- meerschaum/jobs/__init__.py +365 -0
- meerschaum/plugins/__init__.py +6 -6
- meerschaum/utils/daemon/Daemon.py +7 -0
- meerschaum/utils/daemon/RotatingFile.py +5 -2
- meerschaum/utils/daemon/StdinFile.py +12 -2
- meerschaum/utils/daemon/__init__.py +2 -0
- meerschaum/utils/formatting/_jobs.py +52 -16
- meerschaum/utils/misc.py +23 -5
- meerschaum/utils/packages/_packages.py +7 -4
- meerschaum/utils/process.py +9 -9
- meerschaum/utils/venv/__init__.py +2 -2
- {meerschaum-2.3.0.dev1.dist-info → meerschaum-2.3.0rc1.dist-info}/METADATA +14 -17
- {meerschaum-2.3.0.dev1.dist-info → meerschaum-2.3.0rc1.dist-info}/RECORD +55 -51
- meerschaum/utils/jobs/__init__.py +0 -245
- {meerschaum-2.3.0.dev1.dist-info → meerschaum-2.3.0rc1.dist-info}/LICENSE +0 -0
- {meerschaum-2.3.0.dev1.dist-info → meerschaum-2.3.0rc1.dist-info}/NOTICE +0 -0
- {meerschaum-2.3.0.dev1.dist-info → meerschaum-2.3.0rc1.dist-info}/WHEEL +0 -0
- {meerschaum-2.3.0.dev1.dist-info → meerschaum-2.3.0rc1.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.3.0.dev1.dist-info → meerschaum-2.3.0rc1.dist-info}/top_level.txt +0 -0
- {meerschaum-2.3.0.dev1.dist-info → meerschaum-2.3.0rc1.dist-info}/zip-safe +0 -0
meerschaum/actions/stop.py
CHANGED
@@ -27,13 +27,20 @@ def _complete_stop(
|
|
27
27
|
"""
|
28
28
|
Override the default Meerschaum `complete_` function.
|
29
29
|
"""
|
30
|
-
from meerschaum.actions.
|
30
|
+
from meerschaum.actions.delete import _complete_delete_jobs
|
31
|
+
from functools import partial
|
32
|
+
|
31
33
|
if action is None:
|
32
34
|
action = []
|
33
35
|
|
36
|
+
_complete_stop_jobs = partial(
|
37
|
+
_complete_delete_jobs,
|
38
|
+
_get_job_method=('running', 'paused', 'restart'),
|
39
|
+
)
|
40
|
+
|
34
41
|
options = {
|
35
|
-
'job' :
|
36
|
-
'jobs' :
|
42
|
+
'job' : _complete_stop_jobs,
|
43
|
+
'jobs' : _complete_stop_jobs,
|
37
44
|
}
|
38
45
|
|
39
46
|
if (
|
@@ -64,7 +71,7 @@ def _stop_jobs(
|
|
64
71
|
|
65
72
|
To see running processes, run `show jobs`.
|
66
73
|
"""
|
67
|
-
from meerschaum.
|
74
|
+
from meerschaum.jobs import (
|
68
75
|
get_filtered_jobs,
|
69
76
|
get_running_jobs,
|
70
77
|
get_paused_jobs,
|
meerschaum/api/_events.py
CHANGED
@@ -19,7 +19,11 @@ from meerschaum.utils.debug import dprint
|
|
19
19
|
from meerschaum.connectors.poll import retry_connect
|
20
20
|
from meerschaum.utils.warnings import warn
|
21
21
|
from meerschaum._internal.term.tools import is_webterm_running
|
22
|
-
from meerschaum.
|
22
|
+
from meerschaum.jobs import (
|
23
|
+
start_check_jobs_thread,
|
24
|
+
stop_check_jobs_thread,
|
25
|
+
get_executor_keys_from_context,
|
26
|
+
)
|
23
27
|
|
24
28
|
_check_jobs_thread = None
|
25
29
|
|
@@ -47,7 +51,8 @@ async def startup():
|
|
47
51
|
await shutdown()
|
48
52
|
os._exit(1)
|
49
53
|
|
50
|
-
|
54
|
+
if get_executor_keys_from_context() == 'local':
|
55
|
+
start_check_jobs_thread()
|
51
56
|
|
52
57
|
|
53
58
|
@app.on_event("shutdown")
|
@@ -60,7 +65,9 @@ async def shutdown():
|
|
60
65
|
if get_api_connector().type == 'sql':
|
61
66
|
get_api_connector().engine.dispose()
|
62
67
|
|
63
|
-
|
68
|
+
if get_executor_keys_from_context() == 'local':
|
69
|
+
stop_check_jobs_thread()
|
70
|
+
|
64
71
|
from meerschaum.api.routes._actions import _temp_jobs
|
65
72
|
for name, job in _temp_jobs.items():
|
66
73
|
job.delete()
|
meerschaum/api/dash/jobs.py
CHANGED
@@ -16,13 +16,12 @@ from meerschaum.api import CHECK_UPDATE
|
|
16
16
|
dbc = attempt_import('dash_bootstrap_components', lazy=False, check_update=CHECK_UPDATE)
|
17
17
|
html, dcc = import_html(), import_dcc()
|
18
18
|
dateutil_parser = attempt_import('dateutil.parser', check_update=CHECK_UPDATE)
|
19
|
-
from meerschaum.
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
Daemon,
|
19
|
+
from meerschaum.jobs import (
|
20
|
+
get_jobs,
|
21
|
+
get_running_jobs,
|
22
|
+
get_paused_jobs,
|
23
|
+
get_stopped_jobs,
|
24
|
+
Job,
|
26
25
|
)
|
27
26
|
from meerschaum.config import get_config
|
28
27
|
from meerschaum.utils.misc import sorted_dict
|
@@ -38,27 +37,25 @@ def get_jobs_cards(state: WebState):
|
|
38
37
|
"""
|
39
38
|
Build cards and alerts lists for jobs.
|
40
39
|
"""
|
41
|
-
|
42
|
-
jobs = get_jobs()
|
40
|
+
jobs = get_jobs(include_hidden=False)
|
43
41
|
session_id = state['session-store.data'].get('session-id', None)
|
44
42
|
is_authenticated = is_session_authenticated(session_id)
|
45
43
|
|
46
|
-
alert = alert_from_success_tuple(daemons)
|
47
44
|
cards = []
|
48
45
|
|
49
46
|
for name, job in jobs.items():
|
50
47
|
d = job.daemon
|
51
48
|
footer_children = html.Div(
|
52
49
|
build_process_timestamps_children(d),
|
53
|
-
id = {'type': 'process-timestamps-div', 'index':
|
50
|
+
id = {'type': 'process-timestamps-div', 'index': name},
|
54
51
|
)
|
55
52
|
follow_logs_button = dbc.DropdownMenuItem(
|
56
53
|
"Follow logs",
|
57
|
-
id = {'type': 'follow-logs-button', 'index':
|
54
|
+
id = {'type': 'follow-logs-button', 'index': name},
|
58
55
|
)
|
59
56
|
download_logs_button = dbc.DropdownMenuItem(
|
60
57
|
"Download logs",
|
61
|
-
id = {'type': 'job-download-logs-button', 'index':
|
58
|
+
id = {'type': 'job-download-logs-button', 'index': name},
|
62
59
|
)
|
63
60
|
logs_menu_children = (
|
64
61
|
([follow_logs_button] if is_authenticated else []) + [download_logs_button]
|
@@ -66,19 +63,19 @@ def get_jobs_cards(state: WebState):
|
|
66
63
|
header_children = [
|
67
64
|
html.Div(
|
68
65
|
build_status_children(d),
|
69
|
-
id
|
70
|
-
style
|
66
|
+
id={'type': 'manage-job-status-div', 'index': name},
|
67
|
+
style={'float': 'left'},
|
71
68
|
),
|
72
69
|
html.Div(
|
73
70
|
dbc.DropdownMenu(
|
74
71
|
logs_menu_children,
|
75
|
-
label
|
76
|
-
size
|
77
|
-
align_end
|
78
|
-
color
|
79
|
-
menu_variant
|
72
|
+
label="Logs",
|
73
|
+
size="sm",
|
74
|
+
align_end=True,
|
75
|
+
color="secondary",
|
76
|
+
menu_variant='dark',
|
80
77
|
),
|
81
|
-
style
|
78
|
+
style={'float': 'right'},
|
82
79
|
),
|
83
80
|
]
|
84
81
|
|
@@ -87,10 +84,10 @@ def get_jobs_cards(state: WebState):
|
|
87
84
|
html.Div(
|
88
85
|
html.P(
|
89
86
|
d.label,
|
90
|
-
className
|
91
|
-
style
|
87
|
+
className="card-text job-card-text",
|
88
|
+
style={"word-wrap": "break-word"},
|
92
89
|
),
|
93
|
-
style
|
90
|
+
style={"white-space": "pre-wrap"},
|
94
91
|
),
|
95
92
|
html.Div(
|
96
93
|
(
|
@@ -98,9 +95,9 @@ def get_jobs_cards(state: WebState):
|
|
98
95
|
if is_authenticated
|
99
96
|
else []
|
100
97
|
),
|
101
|
-
id
|
98
|
+
id={'type': 'manage-job-buttons-div', 'index': name},
|
102
99
|
),
|
103
|
-
html.Div(id={'type': 'manage-job-alert-div', 'index':
|
100
|
+
html.Div(id={'type': 'manage-job-alert-div', 'index': name}),
|
104
101
|
]
|
105
102
|
|
106
103
|
cards.append(
|
@@ -114,11 +111,11 @@ def get_jobs_cards(state: WebState):
|
|
114
111
|
return cards, []
|
115
112
|
|
116
113
|
|
117
|
-
def build_manage_job_buttons_div_children(
|
114
|
+
def build_manage_job_buttons_div_children(job: Job):
|
118
115
|
"""
|
119
116
|
Return the children for the manage job buttons div.
|
120
117
|
"""
|
121
|
-
buttons = build_manage_job_buttons(
|
118
|
+
buttons = build_manage_job_buttons(job)
|
122
119
|
if not buttons:
|
123
120
|
return []
|
124
121
|
return [
|
@@ -130,96 +127,98 @@ def build_manage_job_buttons_div_children(daemon: Daemon):
|
|
130
127
|
]
|
131
128
|
|
132
129
|
|
133
|
-
def build_manage_job_buttons(
|
130
|
+
def build_manage_job_buttons(job: Job):
|
134
131
|
"""
|
135
|
-
Return the currently available job management buttons for a given
|
132
|
+
Return the currently available job management buttons for a given Job.
|
136
133
|
"""
|
137
|
-
if
|
134
|
+
if job is None:
|
138
135
|
return []
|
136
|
+
|
139
137
|
start_button = dbc.Button(
|
140
138
|
'Start',
|
141
|
-
size
|
142
|
-
color
|
143
|
-
style
|
144
|
-
id
|
139
|
+
size='sm',
|
140
|
+
color='success',
|
141
|
+
style={'width': '100%'},
|
142
|
+
id={
|
145
143
|
'type': 'manage-job-button',
|
146
144
|
'action': 'start',
|
147
|
-
'index':
|
145
|
+
'index': job.name,
|
148
146
|
},
|
149
147
|
)
|
150
148
|
pause_button = dbc.Button(
|
151
149
|
'Pause',
|
152
|
-
size
|
153
|
-
color
|
154
|
-
style
|
155
|
-
id
|
150
|
+
size='sm',
|
151
|
+
color='warning',
|
152
|
+
style={'width': '100%'},
|
153
|
+
id={
|
156
154
|
'type': 'manage-job-button',
|
157
155
|
'action': 'pause',
|
158
|
-
'index':
|
156
|
+
'index': job.name,
|
159
157
|
},
|
160
158
|
)
|
161
159
|
stop_button = dbc.Button(
|
162
160
|
'Stop',
|
163
|
-
size
|
164
|
-
color
|
165
|
-
style
|
166
|
-
id
|
161
|
+
size='sm',
|
162
|
+
color='danger',
|
163
|
+
style={'width': '100%'},
|
164
|
+
id={
|
167
165
|
'type': 'manage-job-button',
|
168
166
|
'action': 'stop',
|
169
|
-
'index':
|
167
|
+
'index': job.name,
|
170
168
|
},
|
171
169
|
)
|
172
170
|
delete_button = dbc.Button(
|
173
171
|
'Delete',
|
174
|
-
size
|
175
|
-
color
|
176
|
-
style
|
177
|
-
id
|
172
|
+
size='sm',
|
173
|
+
color='danger',
|
174
|
+
style={'width': '100%'},
|
175
|
+
id={
|
178
176
|
'type': 'manage-job-button',
|
179
177
|
'action': 'delete',
|
180
|
-
'index':
|
178
|
+
'index': job.name,
|
181
179
|
},
|
182
180
|
)
|
183
181
|
buttons = []
|
184
|
-
if
|
182
|
+
if job.status in ('stopped', 'paused'):
|
185
183
|
buttons.append(start_button)
|
186
|
-
if
|
184
|
+
if job.status == 'stopped':
|
187
185
|
buttons.append(delete_button)
|
188
|
-
if
|
186
|
+
if job.status in ('running',):
|
189
187
|
buttons.append(pause_button)
|
190
|
-
if
|
188
|
+
if job.status in ('running', 'paused'):
|
191
189
|
buttons.append(stop_button)
|
192
190
|
|
193
191
|
return buttons
|
194
192
|
|
195
193
|
|
196
|
-
def build_status_children(
|
194
|
+
def build_status_children(job: Job) -> List[html.P]:
|
197
195
|
"""
|
198
196
|
Return the status HTML component for this daemon.
|
199
197
|
"""
|
200
|
-
if
|
198
|
+
if job is None:
|
201
199
|
return STATUS_EMOJI['dne']
|
202
200
|
|
203
201
|
status_str = (
|
204
|
-
STATUS_EMOJI.get(
|
202
|
+
STATUS_EMOJI.get(job.status, STATUS_EMOJI['stopped'])
|
205
203
|
+ ' '
|
206
|
-
+
|
204
|
+
+ job.status.capitalize()
|
207
205
|
)
|
208
206
|
return html.P(
|
209
207
|
html.B(status_str),
|
210
|
-
className
|
208
|
+
className=f"{job.status}-job",
|
211
209
|
)
|
212
210
|
|
213
211
|
|
214
|
-
def build_process_timestamps_children(
|
212
|
+
def build_process_timestamps_children(job: Job) -> List[dbc.Row]:
|
215
213
|
"""
|
216
214
|
Return the children to the process timestamps in the footer of the job card.
|
217
215
|
"""
|
218
|
-
if
|
216
|
+
if job is None:
|
219
217
|
return []
|
218
|
+
|
220
219
|
children = []
|
221
220
|
for timestamp_key, timestamp_val in sorted_dict(
|
222
|
-
daemon.properties.get('process', {})
|
221
|
+
job.daemon.properties.get('process', {})
|
223
222
|
).items():
|
224
223
|
timestamp = dateutil_parser.parse(timestamp_val)
|
225
224
|
timestamp_str = timestamp.strftime('%Y-%m-%d %H:%M UTC')
|
@@ -229,21 +228,21 @@ def build_process_timestamps_children(daemon: Daemon) -> List[dbc.Row]:
|
|
229
228
|
dbc.Col(
|
230
229
|
html.P(
|
231
230
|
timestamp_key.capitalize(),
|
232
|
-
style
|
233
|
-
className
|
231
|
+
style={'font-size': 'small'},
|
232
|
+
className='text-muted mb-0',
|
234
233
|
),
|
235
|
-
width
|
234
|
+
width=4,
|
236
235
|
),
|
237
236
|
dbc.Col(
|
238
237
|
html.P(
|
239
238
|
timestamp_str,
|
240
|
-
style
|
241
|
-
className
|
239
|
+
style={'font-size': 'small', 'text-align': 'right'},
|
240
|
+
className='text-muted mb-0',
|
242
241
|
),
|
243
|
-
width
|
242
|
+
width=8,
|
244
243
|
),
|
245
244
|
],
|
246
|
-
justify
|
245
|
+
justify='between',
|
247
246
|
)
|
248
247
|
)
|
249
248
|
return children
|
@@ -16,7 +16,7 @@ from datetime import datetime, timezone
|
|
16
16
|
from fastapi import WebSocket, WebSocketDisconnect
|
17
17
|
|
18
18
|
from meerschaum.utils.misc import generate_password
|
19
|
-
from meerschaum.
|
19
|
+
from meerschaum.jobs import Job
|
20
20
|
from meerschaum.utils.warnings import warn
|
21
21
|
from meerschaum.utils.typing import SuccessTuple, Union, List, Dict
|
22
22
|
from meerschaum.api import (
|
@@ -92,6 +92,7 @@ async def do_action_websocket(websocket: WebSocket):
|
|
92
92
|
)
|
93
93
|
|
94
94
|
job = None
|
95
|
+
job_name = '.' + generate_password(12)
|
95
96
|
try:
|
96
97
|
token = await websocket.receive_text()
|
97
98
|
user = await manager.get_current_user(token) if not no_auth else None
|
@@ -101,7 +102,11 @@ async def do_action_websocket(websocket: WebSocket):
|
|
101
102
|
detail="Invalid credentials.",
|
102
103
|
)
|
103
104
|
|
104
|
-
auth_success, auth_msg =
|
105
|
+
auth_success, auth_msg = (
|
106
|
+
is_user_allowed_to_execute(user)
|
107
|
+
if not no_auth
|
108
|
+
else (True, "Success")
|
109
|
+
)
|
105
110
|
auth_payload = {
|
106
111
|
'is_authenticated': auth_success,
|
107
112
|
'timestamp': datetime.now(timezone.utc).isoformat(),
|
@@ -116,10 +121,10 @@ async def do_action_websocket(websocket: WebSocket):
|
|
116
121
|
_ = kwargs.pop('shell', None)
|
117
122
|
sysargs = parse_dict_to_sysargs(kwargs)
|
118
123
|
|
119
|
-
job_name = '.' + generate_password(12)
|
120
124
|
job = Job(
|
121
125
|
job_name,
|
122
126
|
sysargs,
|
127
|
+
executor_keys='local',
|
123
128
|
_properties={
|
124
129
|
'logs': {
|
125
130
|
'write_timestamps': False,
|
meerschaum/api/routes/_jobs.py
CHANGED
@@ -19,7 +19,12 @@ from functools import partial
|
|
19
19
|
from fastapi import WebSocket, WebSocketDisconnect
|
20
20
|
|
21
21
|
from meerschaum.utils.typing import Dict, Any, SuccessTuple, List, Optional, Union
|
22
|
-
from meerschaum.
|
22
|
+
from meerschaum.jobs import (
|
23
|
+
get_jobs as _get_jobs,
|
24
|
+
Job,
|
25
|
+
StopMonitoringLogs,
|
26
|
+
get_executor_keys_from_context,
|
27
|
+
)
|
23
28
|
from meerschaum.utils.warnings import warn
|
24
29
|
|
25
30
|
from meerschaum.api import (
|
@@ -34,6 +39,8 @@ from meerschaum.api import (
|
|
34
39
|
from meerschaum.config.static import STATIC_CONFIG
|
35
40
|
|
36
41
|
JOBS_STDIN_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stdin_message']
|
42
|
+
JOBS_STOP_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stop_message']
|
43
|
+
EXECUTOR_KEYS: str = get_executor_keys_from_context()
|
37
44
|
|
38
45
|
|
39
46
|
@app.get(endpoints['jobs'], tags=['Jobs'])
|
@@ -45,15 +52,21 @@ def get_jobs(
|
|
45
52
|
"""
|
46
53
|
Return metadata about the current jobs.
|
47
54
|
"""
|
48
|
-
jobs = _get_jobs()
|
55
|
+
jobs = _get_jobs(executor_keys=EXECUTOR_KEYS, combine_local_and_systemd=False)
|
49
56
|
return {
|
50
57
|
name: {
|
51
58
|
'sysargs': job.sysargs,
|
52
59
|
'result': job.result,
|
60
|
+
'restart': job.restart,
|
61
|
+
'status': job.status,
|
53
62
|
'daemon': {
|
54
|
-
'status': job.daemon.status,
|
55
|
-
'pid': job.
|
56
|
-
'properties':
|
63
|
+
'status': job.daemon.status if job.executor_keys is None else job.status,
|
64
|
+
'pid': job.pid,
|
65
|
+
'properties': (
|
66
|
+
job.daemon.properties
|
67
|
+
if job.executor is None
|
68
|
+
else job.executor.get_job_properties(name)
|
69
|
+
),
|
57
70
|
},
|
58
71
|
}
|
59
72
|
for name, job in jobs.items()
|
@@ -70,7 +83,7 @@ def get_job(
|
|
70
83
|
"""
|
71
84
|
Return metadata for a single job.
|
72
85
|
"""
|
73
|
-
job = Job(name)
|
86
|
+
job = Job(name, executor_keys=EXECUTOR_KEYS)
|
74
87
|
if not job.exists():
|
75
88
|
raise fastapi.HTTPException(
|
76
89
|
status_code=404,
|
@@ -80,10 +93,16 @@ def get_job(
|
|
80
93
|
return {
|
81
94
|
'sysargs': job.sysargs,
|
82
95
|
'result': job.result,
|
96
|
+
'restart': job.restart,
|
97
|
+
'status': job.status,
|
83
98
|
'daemon': {
|
84
|
-
'status': job.daemon.status,
|
85
|
-
'pid': job.
|
86
|
-
'properties':
|
99
|
+
'status': job.daemon.status if job.executor_keys is None else job.status,
|
100
|
+
'pid': job.pid,
|
101
|
+
'properties': (
|
102
|
+
job.daemon.properties
|
103
|
+
if job.executor is None
|
104
|
+
else job.executor.get_job_properties(job.name)
|
105
|
+
),
|
87
106
|
},
|
88
107
|
}
|
89
108
|
|
@@ -99,7 +118,7 @@ def create_job(
|
|
99
118
|
"""
|
100
119
|
Create and start a new job.
|
101
120
|
"""
|
102
|
-
job = Job(name, sysargs)
|
121
|
+
job = Job(name, sysargs, executor_keys=EXECUTOR_KEYS)
|
103
122
|
if job.exists():
|
104
123
|
raise fastapi.HTTPException(
|
105
124
|
status_code=409,
|
@@ -119,7 +138,7 @@ def delete_job(
|
|
119
138
|
"""
|
120
139
|
Delete a job.
|
121
140
|
"""
|
122
|
-
job = Job(name)
|
141
|
+
job = Job(name, executor_keys=EXECUTOR_KEYS)
|
123
142
|
return job.delete()
|
124
143
|
|
125
144
|
|
@@ -133,7 +152,7 @@ def get_job_exists(
|
|
133
152
|
"""
|
134
153
|
Return whether a job exists.
|
135
154
|
"""
|
136
|
-
job = Job(name)
|
155
|
+
job = Job(name, executor_keys=EXECUTOR_KEYS)
|
137
156
|
return job.exists()
|
138
157
|
|
139
158
|
|
@@ -148,7 +167,7 @@ def get_logs(
|
|
148
167
|
Return a job's log text.
|
149
168
|
To stream log text, connect to the WebSocket endpoint `/logs/{name}/ws`.
|
150
169
|
"""
|
151
|
-
job = Job(name)
|
170
|
+
job = Job(name, executor_keys=EXECUTOR_KEYS)
|
152
171
|
if not job.exists():
|
153
172
|
raise fastapi.HTTPException(
|
154
173
|
status_code=404,
|
@@ -168,7 +187,7 @@ def start_job(
|
|
168
187
|
"""
|
169
188
|
Start a job if stopped.
|
170
189
|
"""
|
171
|
-
job = Job(name)
|
190
|
+
job = Job(name, executor_keys=EXECUTOR_KEYS)
|
172
191
|
if not job.exists():
|
173
192
|
raise fastapi.HTTPException(
|
174
193
|
status_code=404,
|
@@ -187,7 +206,7 @@ def stop_job(
|
|
187
206
|
"""
|
188
207
|
Stop a job if running.
|
189
208
|
"""
|
190
|
-
job = Job(name)
|
209
|
+
job = Job(name, executor_keys=EXECUTOR_KEYS)
|
191
210
|
if not job.exists():
|
192
211
|
raise fastapi.HTTPException(
|
193
212
|
status_code=404,
|
@@ -206,7 +225,7 @@ def pause_job(
|
|
206
225
|
"""
|
207
226
|
Pause a job if running.
|
208
227
|
"""
|
209
|
-
job = Job(name)
|
228
|
+
job = Job(name, executor_keys=EXECUTOR_KEYS)
|
210
229
|
if not job.exists():
|
211
230
|
raise fastapi.HTTPException(
|
212
231
|
status_code=404,
|
@@ -225,7 +244,7 @@ def get_stop_time(
|
|
225
244
|
"""
|
226
245
|
Get the timestamp when the job was manually stopped.
|
227
246
|
"""
|
228
|
-
job = Job(name)
|
247
|
+
job = Job(name, executor_keys=EXECUTOR_KEYS)
|
229
248
|
return job.stop_time
|
230
249
|
|
231
250
|
|
@@ -239,19 +258,16 @@ def get_is_blocking_on_stdin(
|
|
239
258
|
"""
|
240
259
|
Return whether a job is blocking on stdin.
|
241
260
|
"""
|
242
|
-
job = Job(name)
|
261
|
+
job = Job(name, executor_keys=EXECUTOR_KEYS)
|
243
262
|
return job.is_blocking_on_stdin()
|
244
263
|
|
245
264
|
|
246
265
|
_job_clients = defaultdict(lambda: [])
|
247
266
|
_job_stop_events = defaultdict(lambda: asyncio.Event())
|
248
|
-
async def notify_clients(name: str, content: str):
|
267
|
+
async def notify_clients(name: str, websocket: WebSocket, content: str):
|
249
268
|
"""
|
250
269
|
Write the given content to all connected clients.
|
251
270
|
"""
|
252
|
-
if not _job_clients[name]:
|
253
|
-
_job_stop_events[name].set()
|
254
|
-
|
255
271
|
async def _notify_client(client):
|
256
272
|
try:
|
257
273
|
await client.send_text(content)
|
@@ -261,20 +277,14 @@ async def notify_clients(name: str, content: str):
|
|
261
277
|
except Exception:
|
262
278
|
pass
|
263
279
|
|
264
|
-
|
265
|
-
asyncio.create_task(_notify_client(client))
|
266
|
-
for client in _job_clients[name]
|
267
|
-
]
|
268
|
-
await asyncio.wait(notify_tasks)
|
280
|
+
await _notify_client(websocket)
|
269
281
|
|
270
282
|
|
271
|
-
async def get_input_from_clients(name):
|
283
|
+
async def get_input_from_clients(name: str, websocket: WebSocket) -> str:
|
272
284
|
"""
|
273
285
|
When a job is blocking on input, return input from the first client which provides it.
|
274
286
|
"""
|
275
|
-
print('GET INPUT FROM CLIENTS')
|
276
287
|
if not _job_clients[name]:
|
277
|
-
print('NO CLIENTS')
|
278
288
|
return ''
|
279
289
|
|
280
290
|
async def _read_client(client):
|
@@ -299,21 +309,59 @@ async def get_input_from_clients(name):
|
|
299
309
|
return task.result()
|
300
310
|
|
301
311
|
|
312
|
+
async def send_stop_message(name: str, client: WebSocket, result: SuccessTuple):
|
313
|
+
"""
|
314
|
+
Send a stop message to clients when the job stops.
|
315
|
+
"""
|
316
|
+
try:
|
317
|
+
await client.send_text(JOBS_STOP_MESSAGE)
|
318
|
+
await client.send_json(result)
|
319
|
+
except WebSocketDisconnect:
|
320
|
+
_job_stop_events[name].set()
|
321
|
+
if client in _job_clients[name]:
|
322
|
+
_job_clients[name].remove(client)
|
323
|
+
except RuntimeError:
|
324
|
+
pass
|
325
|
+
except Exception:
|
326
|
+
warn(traceback.format_exc())
|
327
|
+
|
328
|
+
|
302
329
|
@app.websocket(endpoints['logs'] + '/{name}/ws')
|
303
330
|
async def logs_websocket(name: str, websocket: WebSocket):
|
304
331
|
"""
|
305
332
|
Stream logs from a job over a websocket.
|
306
333
|
"""
|
307
334
|
await websocket.accept()
|
308
|
-
job = Job(name)
|
335
|
+
job = Job(name, executor_keys=EXECUTOR_KEYS)
|
309
336
|
_job_clients[name].append(websocket)
|
310
337
|
|
311
338
|
async def monitor_logs():
|
312
|
-
|
313
|
-
partial(
|
314
|
-
|
315
|
-
|
316
|
-
|
339
|
+
try:
|
340
|
+
callback_function = partial(
|
341
|
+
notify_clients,
|
342
|
+
name,
|
343
|
+
websocket,
|
344
|
+
)
|
345
|
+
input_callback_function = partial(
|
346
|
+
get_input_from_clients,
|
347
|
+
name,
|
348
|
+
websocket,
|
349
|
+
)
|
350
|
+
stop_callback_function = partial(
|
351
|
+
send_stop_message,
|
352
|
+
name,
|
353
|
+
websocket,
|
354
|
+
)
|
355
|
+
await job.monitor_logs_async(
|
356
|
+
callback_function=callback_function,
|
357
|
+
input_callback_function=input_callback_function,
|
358
|
+
stop_callback_function=stop_callback_function,
|
359
|
+
stop_event=_job_stop_events[name],
|
360
|
+
stop_on_exit=True,
|
361
|
+
accept_input=True,
|
362
|
+
)
|
363
|
+
except Exception:
|
364
|
+
warn(traceback.format_exc())
|
317
365
|
|
318
366
|
try:
|
319
367
|
token = await websocket.receive_text()
|
@@ -329,7 +377,8 @@ async def logs_websocket(name: str, websocket: WebSocket):
|
|
329
377
|
await websocket.send_text("Invalid credentials.")
|
330
378
|
await websocket.close()
|
331
379
|
except WebSocketDisconnect:
|
332
|
-
|
380
|
+
_job_stop_events[name].set()
|
381
|
+
monitor_task.cancel()
|
333
382
|
except asyncio.CancelledError:
|
334
383
|
pass
|
335
384
|
except Exception:
|
meerschaum/config/_default.py
CHANGED
@@ -16,7 +16,7 @@ default_meerschaum_config = {
|
|
16
16
|
'api_instance': 'MRSM{meerschaum:instance}',
|
17
17
|
'web_instance': 'MRSM{meerschaum:instance}',
|
18
18
|
'default_repository': 'api:mrsm',
|
19
|
-
'default_executor': 'local',
|
19
|
+
# 'default_executor': 'local',
|
20
20
|
'connectors': {
|
21
21
|
'sql': {
|
22
22
|
'default': {},
|
meerschaum/config/_paths.py
CHANGED
@@ -181,6 +181,11 @@ paths = {
|
|
181
181
|
'LOGS_RESOURCES_PATH' : ('{ROOT_DIR_PATH}', 'logs'),
|
182
182
|
'DAEMON_ERROR_LOG_PATH' : ('{ROOT_DIR_PATH}', 'daemon_errors.log'),
|
183
183
|
'CHECK_JOBS_LOCK_PATH' : ('{INTERNAL_RESOURCES_PATH}', 'check-jobs.lock'),
|
184
|
+
|
185
|
+
'SYSTEMD_RESOURCES_PATH' : ('{DOT_CONFIG_DIR_PATH}', 'systemd'),
|
186
|
+
'SYSTEMD_USER_RESOURCES_PATH' : ('{SYSTEMD_RESOURCES_PATH}', 'user'),
|
187
|
+
'SYSTEMD_ROOT_RESOURCES_PATH' : ('{ROOT_DIR_PATH}', 'systemd'),
|
188
|
+
'SYSTEMD_LOGS_RESOURCES_PATH' : ('{SYSTEMD_ROOT_RESOURCES_PATH}', 'logs'),
|
184
189
|
}
|
185
190
|
|
186
191
|
def set_root(root: Union[Path, str]):
|