meerschaum 2.3.0.dev3__py3-none-any.whl → 2.3.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/__init__.py +3 -2
- meerschaum/__main__.py +0 -5
- meerschaum/_internal/arguments/_parser.py +6 -2
- meerschaum/_internal/entry.py +36 -6
- meerschaum/_internal/shell/Shell.py +32 -20
- meerschaum/actions/attach.py +12 -7
- meerschaum/actions/copy.py +68 -41
- meerschaum/actions/delete.py +64 -21
- meerschaum/actions/edit.py +3 -3
- meerschaum/actions/install.py +40 -32
- meerschaum/actions/pause.py +44 -27
- meerschaum/actions/restart.py +107 -0
- meerschaum/actions/show.py +8 -8
- meerschaum/actions/start.py +25 -40
- meerschaum/actions/stop.py +11 -4
- meerschaum/api/_events.py +10 -3
- meerschaum/api/dash/jobs.py +69 -70
- meerschaum/api/routes/_actions.py +8 -3
- meerschaum/api/routes/_jobs.py +37 -19
- meerschaum/config/_default.py +1 -1
- meerschaum/config/_paths.py +5 -0
- meerschaum/config/_version.py +1 -1
- meerschaum/config/static/__init__.py +3 -0
- meerschaum/connectors/Connector.py +13 -7
- meerschaum/connectors/__init__.py +21 -5
- meerschaum/connectors/api/APIConnector.py +3 -0
- meerschaum/connectors/api/_jobs.py +30 -3
- meerschaum/connectors/parse.py +10 -13
- meerschaum/core/Pipe/_bootstrap.py +16 -8
- meerschaum/jobs/_Executor.py +69 -0
- meerschaum/{utils/jobs → jobs}/_Job.py +160 -20
- meerschaum/jobs/_LocalExecutor.py +88 -0
- meerschaum/jobs/_SystemdExecutor.py +608 -0
- meerschaum/jobs/__init__.py +365 -0
- meerschaum/plugins/__init__.py +6 -6
- meerschaum/utils/daemon/Daemon.py +7 -0
- meerschaum/utils/daemon/RotatingFile.py +5 -2
- meerschaum/utils/daemon/StdinFile.py +12 -2
- meerschaum/utils/daemon/__init__.py +2 -0
- meerschaum/utils/formatting/_jobs.py +52 -16
- meerschaum/utils/misc.py +23 -5
- meerschaum/utils/packages/_packages.py +7 -4
- meerschaum/utils/process.py +9 -9
- meerschaum/utils/venv/__init__.py +2 -2
- {meerschaum-2.3.0.dev3.dist-info → meerschaum-2.3.0rc1.dist-info}/METADATA +14 -17
- {meerschaum-2.3.0.dev3.dist-info → meerschaum-2.3.0rc1.dist-info}/RECORD +52 -48
- meerschaum/utils/jobs/__init__.py +0 -245
- {meerschaum-2.3.0.dev3.dist-info → meerschaum-2.3.0rc1.dist-info}/LICENSE +0 -0
- {meerschaum-2.3.0.dev3.dist-info → meerschaum-2.3.0rc1.dist-info}/NOTICE +0 -0
- {meerschaum-2.3.0.dev3.dist-info → meerschaum-2.3.0rc1.dist-info}/WHEEL +0 -0
- {meerschaum-2.3.0.dev3.dist-info → meerschaum-2.3.0rc1.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.3.0.dev3.dist-info → meerschaum-2.3.0rc1.dist-info}/top_level.txt +0 -0
- {meerschaum-2.3.0.dev3.dist-info → meerschaum-2.3.0rc1.dist-info}/zip-safe +0 -0
meerschaum/api/dash/jobs.py
CHANGED
@@ -16,13 +16,12 @@ from meerschaum.api import CHECK_UPDATE
|
|
16
16
|
dbc = attempt_import('dash_bootstrap_components', lazy=False, check_update=CHECK_UPDATE)
|
17
17
|
html, dcc = import_html(), import_dcc()
|
18
18
|
dateutil_parser = attempt_import('dateutil.parser', check_update=CHECK_UPDATE)
|
19
|
-
from meerschaum.
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
Daemon,
|
19
|
+
from meerschaum.jobs import (
|
20
|
+
get_jobs,
|
21
|
+
get_running_jobs,
|
22
|
+
get_paused_jobs,
|
23
|
+
get_stopped_jobs,
|
24
|
+
Job,
|
26
25
|
)
|
27
26
|
from meerschaum.config import get_config
|
28
27
|
from meerschaum.utils.misc import sorted_dict
|
@@ -38,27 +37,25 @@ def get_jobs_cards(state: WebState):
|
|
38
37
|
"""
|
39
38
|
Build cards and alerts lists for jobs.
|
40
39
|
"""
|
41
|
-
|
42
|
-
jobs = get_jobs()
|
40
|
+
jobs = get_jobs(include_hidden=False)
|
43
41
|
session_id = state['session-store.data'].get('session-id', None)
|
44
42
|
is_authenticated = is_session_authenticated(session_id)
|
45
43
|
|
46
|
-
alert = alert_from_success_tuple(daemons)
|
47
44
|
cards = []
|
48
45
|
|
49
46
|
for name, job in jobs.items():
|
50
47
|
d = job.daemon
|
51
48
|
footer_children = html.Div(
|
52
49
|
build_process_timestamps_children(d),
|
53
|
-
id = {'type': 'process-timestamps-div', 'index':
|
50
|
+
id = {'type': 'process-timestamps-div', 'index': name},
|
54
51
|
)
|
55
52
|
follow_logs_button = dbc.DropdownMenuItem(
|
56
53
|
"Follow logs",
|
57
|
-
id = {'type': 'follow-logs-button', 'index':
|
54
|
+
id = {'type': 'follow-logs-button', 'index': name},
|
58
55
|
)
|
59
56
|
download_logs_button = dbc.DropdownMenuItem(
|
60
57
|
"Download logs",
|
61
|
-
id = {'type': 'job-download-logs-button', 'index':
|
58
|
+
id = {'type': 'job-download-logs-button', 'index': name},
|
62
59
|
)
|
63
60
|
logs_menu_children = (
|
64
61
|
([follow_logs_button] if is_authenticated else []) + [download_logs_button]
|
@@ -66,19 +63,19 @@ def get_jobs_cards(state: WebState):
|
|
66
63
|
header_children = [
|
67
64
|
html.Div(
|
68
65
|
build_status_children(d),
|
69
|
-
id
|
70
|
-
style
|
66
|
+
id={'type': 'manage-job-status-div', 'index': name},
|
67
|
+
style={'float': 'left'},
|
71
68
|
),
|
72
69
|
html.Div(
|
73
70
|
dbc.DropdownMenu(
|
74
71
|
logs_menu_children,
|
75
|
-
label
|
76
|
-
size
|
77
|
-
align_end
|
78
|
-
color
|
79
|
-
menu_variant
|
72
|
+
label="Logs",
|
73
|
+
size="sm",
|
74
|
+
align_end=True,
|
75
|
+
color="secondary",
|
76
|
+
menu_variant='dark',
|
80
77
|
),
|
81
|
-
style
|
78
|
+
style={'float': 'right'},
|
82
79
|
),
|
83
80
|
]
|
84
81
|
|
@@ -87,10 +84,10 @@ def get_jobs_cards(state: WebState):
|
|
87
84
|
html.Div(
|
88
85
|
html.P(
|
89
86
|
d.label,
|
90
|
-
className
|
91
|
-
style
|
87
|
+
className="card-text job-card-text",
|
88
|
+
style={"word-wrap": "break-word"},
|
92
89
|
),
|
93
|
-
style
|
90
|
+
style={"white-space": "pre-wrap"},
|
94
91
|
),
|
95
92
|
html.Div(
|
96
93
|
(
|
@@ -98,9 +95,9 @@ def get_jobs_cards(state: WebState):
|
|
98
95
|
if is_authenticated
|
99
96
|
else []
|
100
97
|
),
|
101
|
-
id
|
98
|
+
id={'type': 'manage-job-buttons-div', 'index': name},
|
102
99
|
),
|
103
|
-
html.Div(id={'type': 'manage-job-alert-div', 'index':
|
100
|
+
html.Div(id={'type': 'manage-job-alert-div', 'index': name}),
|
104
101
|
]
|
105
102
|
|
106
103
|
cards.append(
|
@@ -114,11 +111,11 @@ def get_jobs_cards(state: WebState):
|
|
114
111
|
return cards, []
|
115
112
|
|
116
113
|
|
117
|
-
def build_manage_job_buttons_div_children(
|
114
|
+
def build_manage_job_buttons_div_children(job: Job):
|
118
115
|
"""
|
119
116
|
Return the children for the manage job buttons div.
|
120
117
|
"""
|
121
|
-
buttons = build_manage_job_buttons(
|
118
|
+
buttons = build_manage_job_buttons(job)
|
122
119
|
if not buttons:
|
123
120
|
return []
|
124
121
|
return [
|
@@ -130,96 +127,98 @@ def build_manage_job_buttons_div_children(daemon: Daemon):
|
|
130
127
|
]
|
131
128
|
|
132
129
|
|
133
|
-
def build_manage_job_buttons(
|
130
|
+
def build_manage_job_buttons(job: Job):
|
134
131
|
"""
|
135
|
-
Return the currently available job management buttons for a given
|
132
|
+
Return the currently available job management buttons for a given Job.
|
136
133
|
"""
|
137
|
-
if
|
134
|
+
if job is None:
|
138
135
|
return []
|
136
|
+
|
139
137
|
start_button = dbc.Button(
|
140
138
|
'Start',
|
141
|
-
size
|
142
|
-
color
|
143
|
-
style
|
144
|
-
id
|
139
|
+
size='sm',
|
140
|
+
color='success',
|
141
|
+
style={'width': '100%'},
|
142
|
+
id={
|
145
143
|
'type': 'manage-job-button',
|
146
144
|
'action': 'start',
|
147
|
-
'index':
|
145
|
+
'index': job.name,
|
148
146
|
},
|
149
147
|
)
|
150
148
|
pause_button = dbc.Button(
|
151
149
|
'Pause',
|
152
|
-
size
|
153
|
-
color
|
154
|
-
style
|
155
|
-
id
|
150
|
+
size='sm',
|
151
|
+
color='warning',
|
152
|
+
style={'width': '100%'},
|
153
|
+
id={
|
156
154
|
'type': 'manage-job-button',
|
157
155
|
'action': 'pause',
|
158
|
-
'index':
|
156
|
+
'index': job.name,
|
159
157
|
},
|
160
158
|
)
|
161
159
|
stop_button = dbc.Button(
|
162
160
|
'Stop',
|
163
|
-
size
|
164
|
-
color
|
165
|
-
style
|
166
|
-
id
|
161
|
+
size='sm',
|
162
|
+
color='danger',
|
163
|
+
style={'width': '100%'},
|
164
|
+
id={
|
167
165
|
'type': 'manage-job-button',
|
168
166
|
'action': 'stop',
|
169
|
-
'index':
|
167
|
+
'index': job.name,
|
170
168
|
},
|
171
169
|
)
|
172
170
|
delete_button = dbc.Button(
|
173
171
|
'Delete',
|
174
|
-
size
|
175
|
-
color
|
176
|
-
style
|
177
|
-
id
|
172
|
+
size='sm',
|
173
|
+
color='danger',
|
174
|
+
style={'width': '100%'},
|
175
|
+
id={
|
178
176
|
'type': 'manage-job-button',
|
179
177
|
'action': 'delete',
|
180
|
-
'index':
|
178
|
+
'index': job.name,
|
181
179
|
},
|
182
180
|
)
|
183
181
|
buttons = []
|
184
|
-
if
|
182
|
+
if job.status in ('stopped', 'paused'):
|
185
183
|
buttons.append(start_button)
|
186
|
-
if
|
184
|
+
if job.status == 'stopped':
|
187
185
|
buttons.append(delete_button)
|
188
|
-
if
|
186
|
+
if job.status in ('running',):
|
189
187
|
buttons.append(pause_button)
|
190
|
-
if
|
188
|
+
if job.status in ('running', 'paused'):
|
191
189
|
buttons.append(stop_button)
|
192
190
|
|
193
191
|
return buttons
|
194
192
|
|
195
193
|
|
196
|
-
def build_status_children(
|
194
|
+
def build_status_children(job: Job) -> List[html.P]:
|
197
195
|
"""
|
198
196
|
Return the status HTML component for this daemon.
|
199
197
|
"""
|
200
|
-
if
|
198
|
+
if job is None:
|
201
199
|
return STATUS_EMOJI['dne']
|
202
200
|
|
203
201
|
status_str = (
|
204
|
-
STATUS_EMOJI.get(
|
202
|
+
STATUS_EMOJI.get(job.status, STATUS_EMOJI['stopped'])
|
205
203
|
+ ' '
|
206
|
-
+
|
204
|
+
+ job.status.capitalize()
|
207
205
|
)
|
208
206
|
return html.P(
|
209
207
|
html.B(status_str),
|
210
|
-
className
|
208
|
+
className=f"{job.status}-job",
|
211
209
|
)
|
212
210
|
|
213
211
|
|
214
|
-
def build_process_timestamps_children(
|
212
|
+
def build_process_timestamps_children(job: Job) -> List[dbc.Row]:
|
215
213
|
"""
|
216
214
|
Return the children to the process timestamps in the footer of the job card.
|
217
215
|
"""
|
218
|
-
if
|
216
|
+
if job is None:
|
219
217
|
return []
|
218
|
+
|
220
219
|
children = []
|
221
220
|
for timestamp_key, timestamp_val in sorted_dict(
|
222
|
-
daemon.properties.get('process', {})
|
221
|
+
job.daemon.properties.get('process', {})
|
223
222
|
).items():
|
224
223
|
timestamp = dateutil_parser.parse(timestamp_val)
|
225
224
|
timestamp_str = timestamp.strftime('%Y-%m-%d %H:%M UTC')
|
@@ -229,21 +228,21 @@ def build_process_timestamps_children(daemon: Daemon) -> List[dbc.Row]:
|
|
229
228
|
dbc.Col(
|
230
229
|
html.P(
|
231
230
|
timestamp_key.capitalize(),
|
232
|
-
style
|
233
|
-
className
|
231
|
+
style={'font-size': 'small'},
|
232
|
+
className='text-muted mb-0',
|
234
233
|
),
|
235
|
-
width
|
234
|
+
width=4,
|
236
235
|
),
|
237
236
|
dbc.Col(
|
238
237
|
html.P(
|
239
238
|
timestamp_str,
|
240
|
-
style
|
241
|
-
className
|
239
|
+
style={'font-size': 'small', 'text-align': 'right'},
|
240
|
+
className='text-muted mb-0',
|
242
241
|
),
|
243
|
-
width
|
242
|
+
width=8,
|
244
243
|
),
|
245
244
|
],
|
246
|
-
justify
|
245
|
+
justify='between',
|
247
246
|
)
|
248
247
|
)
|
249
248
|
return children
|
@@ -16,7 +16,7 @@ from datetime import datetime, timezone
|
|
16
16
|
from fastapi import WebSocket, WebSocketDisconnect
|
17
17
|
|
18
18
|
from meerschaum.utils.misc import generate_password
|
19
|
-
from meerschaum.
|
19
|
+
from meerschaum.jobs import Job
|
20
20
|
from meerschaum.utils.warnings import warn
|
21
21
|
from meerschaum.utils.typing import SuccessTuple, Union, List, Dict
|
22
22
|
from meerschaum.api import (
|
@@ -92,6 +92,7 @@ async def do_action_websocket(websocket: WebSocket):
|
|
92
92
|
)
|
93
93
|
|
94
94
|
job = None
|
95
|
+
job_name = '.' + generate_password(12)
|
95
96
|
try:
|
96
97
|
token = await websocket.receive_text()
|
97
98
|
user = await manager.get_current_user(token) if not no_auth else None
|
@@ -101,7 +102,11 @@ async def do_action_websocket(websocket: WebSocket):
|
|
101
102
|
detail="Invalid credentials.",
|
102
103
|
)
|
103
104
|
|
104
|
-
auth_success, auth_msg =
|
105
|
+
auth_success, auth_msg = (
|
106
|
+
is_user_allowed_to_execute(user)
|
107
|
+
if not no_auth
|
108
|
+
else (True, "Success")
|
109
|
+
)
|
105
110
|
auth_payload = {
|
106
111
|
'is_authenticated': auth_success,
|
107
112
|
'timestamp': datetime.now(timezone.utc).isoformat(),
|
@@ -116,10 +121,10 @@ async def do_action_websocket(websocket: WebSocket):
|
|
116
121
|
_ = kwargs.pop('shell', None)
|
117
122
|
sysargs = parse_dict_to_sysargs(kwargs)
|
118
123
|
|
119
|
-
job_name = '.' + generate_password(12)
|
120
124
|
job = Job(
|
121
125
|
job_name,
|
122
126
|
sysargs,
|
127
|
+
executor_keys='local',
|
123
128
|
_properties={
|
124
129
|
'logs': {
|
125
130
|
'write_timestamps': False,
|
meerschaum/api/routes/_jobs.py
CHANGED
@@ -19,7 +19,12 @@ from functools import partial
|
|
19
19
|
from fastapi import WebSocket, WebSocketDisconnect
|
20
20
|
|
21
21
|
from meerschaum.utils.typing import Dict, Any, SuccessTuple, List, Optional, Union
|
22
|
-
from meerschaum.
|
22
|
+
from meerschaum.jobs import (
|
23
|
+
get_jobs as _get_jobs,
|
24
|
+
Job,
|
25
|
+
StopMonitoringLogs,
|
26
|
+
get_executor_keys_from_context,
|
27
|
+
)
|
23
28
|
from meerschaum.utils.warnings import warn
|
24
29
|
|
25
30
|
from meerschaum.api import (
|
@@ -35,6 +40,7 @@ from meerschaum.config.static import STATIC_CONFIG
|
|
35
40
|
|
36
41
|
JOBS_STDIN_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stdin_message']
|
37
42
|
JOBS_STOP_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stop_message']
|
43
|
+
EXECUTOR_KEYS: str = get_executor_keys_from_context()
|
38
44
|
|
39
45
|
|
40
46
|
@app.get(endpoints['jobs'], tags=['Jobs'])
|
@@ -46,15 +52,21 @@ def get_jobs(
|
|
46
52
|
"""
|
47
53
|
Return metadata about the current jobs.
|
48
54
|
"""
|
49
|
-
jobs = _get_jobs()
|
55
|
+
jobs = _get_jobs(executor_keys=EXECUTOR_KEYS, combine_local_and_systemd=False)
|
50
56
|
return {
|
51
57
|
name: {
|
52
58
|
'sysargs': job.sysargs,
|
53
59
|
'result': job.result,
|
60
|
+
'restart': job.restart,
|
61
|
+
'status': job.status,
|
54
62
|
'daemon': {
|
55
|
-
'status': job.daemon.status,
|
56
|
-
'pid': job.
|
57
|
-
'properties':
|
63
|
+
'status': job.daemon.status if job.executor_keys is None else job.status,
|
64
|
+
'pid': job.pid,
|
65
|
+
'properties': (
|
66
|
+
job.daemon.properties
|
67
|
+
if job.executor is None
|
68
|
+
else job.executor.get_job_properties(name)
|
69
|
+
),
|
58
70
|
},
|
59
71
|
}
|
60
72
|
for name, job in jobs.items()
|
@@ -71,7 +83,7 @@ def get_job(
|
|
71
83
|
"""
|
72
84
|
Return metadata for a single job.
|
73
85
|
"""
|
74
|
-
job = Job(name)
|
86
|
+
job = Job(name, executor_keys=EXECUTOR_KEYS)
|
75
87
|
if not job.exists():
|
76
88
|
raise fastapi.HTTPException(
|
77
89
|
status_code=404,
|
@@ -81,10 +93,16 @@ def get_job(
|
|
81
93
|
return {
|
82
94
|
'sysargs': job.sysargs,
|
83
95
|
'result': job.result,
|
96
|
+
'restart': job.restart,
|
97
|
+
'status': job.status,
|
84
98
|
'daemon': {
|
85
|
-
'status': job.daemon.status,
|
86
|
-
'pid': job.
|
87
|
-
'properties':
|
99
|
+
'status': job.daemon.status if job.executor_keys is None else job.status,
|
100
|
+
'pid': job.pid,
|
101
|
+
'properties': (
|
102
|
+
job.daemon.properties
|
103
|
+
if job.executor is None
|
104
|
+
else job.executor.get_job_properties(job.name)
|
105
|
+
),
|
88
106
|
},
|
89
107
|
}
|
90
108
|
|
@@ -100,7 +118,7 @@ def create_job(
|
|
100
118
|
"""
|
101
119
|
Create and start a new job.
|
102
120
|
"""
|
103
|
-
job = Job(name, sysargs)
|
121
|
+
job = Job(name, sysargs, executor_keys=EXECUTOR_KEYS)
|
104
122
|
if job.exists():
|
105
123
|
raise fastapi.HTTPException(
|
106
124
|
status_code=409,
|
@@ -120,7 +138,7 @@ def delete_job(
|
|
120
138
|
"""
|
121
139
|
Delete a job.
|
122
140
|
"""
|
123
|
-
job = Job(name)
|
141
|
+
job = Job(name, executor_keys=EXECUTOR_KEYS)
|
124
142
|
return job.delete()
|
125
143
|
|
126
144
|
|
@@ -134,7 +152,7 @@ def get_job_exists(
|
|
134
152
|
"""
|
135
153
|
Return whether a job exists.
|
136
154
|
"""
|
137
|
-
job = Job(name)
|
155
|
+
job = Job(name, executor_keys=EXECUTOR_KEYS)
|
138
156
|
return job.exists()
|
139
157
|
|
140
158
|
|
@@ -149,7 +167,7 @@ def get_logs(
|
|
149
167
|
Return a job's log text.
|
150
168
|
To stream log text, connect to the WebSocket endpoint `/logs/{name}/ws`.
|
151
169
|
"""
|
152
|
-
job = Job(name)
|
170
|
+
job = Job(name, executor_keys=EXECUTOR_KEYS)
|
153
171
|
if not job.exists():
|
154
172
|
raise fastapi.HTTPException(
|
155
173
|
status_code=404,
|
@@ -169,7 +187,7 @@ def start_job(
|
|
169
187
|
"""
|
170
188
|
Start a job if stopped.
|
171
189
|
"""
|
172
|
-
job = Job(name)
|
190
|
+
job = Job(name, executor_keys=EXECUTOR_KEYS)
|
173
191
|
if not job.exists():
|
174
192
|
raise fastapi.HTTPException(
|
175
193
|
status_code=404,
|
@@ -188,7 +206,7 @@ def stop_job(
|
|
188
206
|
"""
|
189
207
|
Stop a job if running.
|
190
208
|
"""
|
191
|
-
job = Job(name)
|
209
|
+
job = Job(name, executor_keys=EXECUTOR_KEYS)
|
192
210
|
if not job.exists():
|
193
211
|
raise fastapi.HTTPException(
|
194
212
|
status_code=404,
|
@@ -207,7 +225,7 @@ def pause_job(
|
|
207
225
|
"""
|
208
226
|
Pause a job if running.
|
209
227
|
"""
|
210
|
-
job = Job(name)
|
228
|
+
job = Job(name, executor_keys=EXECUTOR_KEYS)
|
211
229
|
if not job.exists():
|
212
230
|
raise fastapi.HTTPException(
|
213
231
|
status_code=404,
|
@@ -226,7 +244,7 @@ def get_stop_time(
|
|
226
244
|
"""
|
227
245
|
Get the timestamp when the job was manually stopped.
|
228
246
|
"""
|
229
|
-
job = Job(name)
|
247
|
+
job = Job(name, executor_keys=EXECUTOR_KEYS)
|
230
248
|
return job.stop_time
|
231
249
|
|
232
250
|
|
@@ -240,7 +258,7 @@ def get_is_blocking_on_stdin(
|
|
240
258
|
"""
|
241
259
|
Return whether a job is blocking on stdin.
|
242
260
|
"""
|
243
|
-
job = Job(name)
|
261
|
+
job = Job(name, executor_keys=EXECUTOR_KEYS)
|
244
262
|
return job.is_blocking_on_stdin()
|
245
263
|
|
246
264
|
|
@@ -314,7 +332,7 @@ async def logs_websocket(name: str, websocket: WebSocket):
|
|
314
332
|
Stream logs from a job over a websocket.
|
315
333
|
"""
|
316
334
|
await websocket.accept()
|
317
|
-
job = Job(name)
|
335
|
+
job = Job(name, executor_keys=EXECUTOR_KEYS)
|
318
336
|
_job_clients[name].append(websocket)
|
319
337
|
|
320
338
|
async def monitor_logs():
|
meerschaum/config/_default.py
CHANGED
@@ -16,7 +16,7 @@ default_meerschaum_config = {
|
|
16
16
|
'api_instance': 'MRSM{meerschaum:instance}',
|
17
17
|
'web_instance': 'MRSM{meerschaum:instance}',
|
18
18
|
'default_repository': 'api:mrsm',
|
19
|
-
'default_executor': 'local',
|
19
|
+
# 'default_executor': 'local',
|
20
20
|
'connectors': {
|
21
21
|
'sql': {
|
22
22
|
'default': {},
|
meerschaum/config/_paths.py
CHANGED
@@ -181,6 +181,11 @@ paths = {
|
|
181
181
|
'LOGS_RESOURCES_PATH' : ('{ROOT_DIR_PATH}', 'logs'),
|
182
182
|
'DAEMON_ERROR_LOG_PATH' : ('{ROOT_DIR_PATH}', 'daemon_errors.log'),
|
183
183
|
'CHECK_JOBS_LOCK_PATH' : ('{INTERNAL_RESOURCES_PATH}', 'check-jobs.lock'),
|
184
|
+
|
185
|
+
'SYSTEMD_RESOURCES_PATH' : ('{DOT_CONFIG_DIR_PATH}', 'systemd'),
|
186
|
+
'SYSTEMD_USER_RESOURCES_PATH' : ('{SYSTEMD_RESOURCES_PATH}', 'user'),
|
187
|
+
'SYSTEMD_ROOT_RESOURCES_PATH' : ('{ROOT_DIR_PATH}', 'systemd'),
|
188
|
+
'SYSTEMD_LOGS_RESOURCES_PATH' : ('{SYSTEMD_ROOT_RESOURCES_PATH}', 'logs'),
|
184
189
|
}
|
185
190
|
|
186
191
|
def set_root(root: Union[Path, str]):
|
meerschaum/config/_version.py
CHANGED
@@ -70,6 +70,9 @@ STATIC_CONFIG: Dict[str, Any] = {
|
|
70
70
|
'noask': 'MRSM_NOASK',
|
71
71
|
'id': 'MRSM_SERVER_ID',
|
72
72
|
'daemon_id': 'MRSM_DAEMON_ID',
|
73
|
+
'systemd_log_path': 'MRSM_SYSTEMD_LOG_PATH',
|
74
|
+
'systemd_stdin_path': 'MRSM_SYSTEMD_STDIN_PATH',
|
75
|
+
'systemd_result_path': 'MRSM_SYSTEMD_RESULT_PATH',
|
73
76
|
'uri_regex': r'MRSM_([a-zA-Z0-9]*)_(\d*[a-zA-Z][a-zA-Z0-9-_+]*$)',
|
74
77
|
'prefix': 'MRSM_',
|
75
78
|
},
|
@@ -21,11 +21,11 @@ class Connector(metaclass=abc.ABCMeta):
|
|
21
21
|
The base connector class to hold connection attributes.
|
22
22
|
"""
|
23
23
|
def __init__(
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
24
|
+
self,
|
25
|
+
type: Optional[str] = None,
|
26
|
+
label: Optional[str] = None,
|
27
|
+
**kw: Any
|
28
|
+
):
|
29
29
|
"""
|
30
30
|
Set the given keyword arguments as attributes.
|
31
31
|
|
@@ -101,7 +101,7 @@ class Connector(metaclass=abc.ABCMeta):
|
|
101
101
|
|
102
102
|
### load user config into self._attributes
|
103
103
|
if self.type in conn_configs and self.label in conn_configs[self.type]:
|
104
|
-
self._attributes.update(conn_configs[self.type][self.label])
|
104
|
+
self._attributes.update(conn_configs[self.type][self.label] or {})
|
105
105
|
|
106
106
|
### load system config into self._sys_config
|
107
107
|
### (deep copy so future Connectors don't inherit changes)
|
@@ -200,7 +200,13 @@ class Connector(metaclass=abc.ABCMeta):
|
|
200
200
|
_type = self.__dict__.get('type', None)
|
201
201
|
if _type is None:
|
202
202
|
import re
|
203
|
-
|
203
|
+
is_executor = self.__class__.__name__.lower().endswith('executor')
|
204
|
+
suffix_regex = (
|
205
|
+
r'connector$'
|
206
|
+
if not is_executor
|
207
|
+
else r'executor$'
|
208
|
+
)
|
209
|
+
_type = re.sub(suffix_regex, '', self.__class__.__name__.lower())
|
204
210
|
self.__dict__['type'] = _type
|
205
211
|
return _type
|
206
212
|
|
@@ -36,9 +36,9 @@ __all__ = (
|
|
36
36
|
### store connectors partitioned by
|
37
37
|
### type, label for reuse
|
38
38
|
connectors: Dict[str, Dict[str, Connector]] = {
|
39
|
-
'api'
|
40
|
-
'sql'
|
41
|
-
'plugin': {},
|
39
|
+
'api' : {},
|
40
|
+
'sql' : {},
|
41
|
+
'plugin' : {},
|
42
42
|
}
|
43
43
|
instance_types: List[str] = ['sql', 'api']
|
44
44
|
_locks: Dict[str, RLock] = {
|
@@ -127,10 +127,13 @@ def get_connector(
|
|
127
127
|
global _loaded_plugin_connectors
|
128
128
|
if isinstance(type, str) and not label and ':' in type:
|
129
129
|
type, label = type.split(':', maxsplit=1)
|
130
|
+
|
130
131
|
with _locks['_loaded_plugin_connectors']:
|
131
132
|
if not _loaded_plugin_connectors:
|
132
133
|
load_plugin_connectors()
|
134
|
+
_load_builtin_custom_connectors()
|
133
135
|
_loaded_plugin_connectors = True
|
136
|
+
|
134
137
|
if type is None and label is None:
|
135
138
|
default_instance_keys = get_config('meerschaum', 'instance', patch=True)
|
136
139
|
### recursive call to get_connector
|
@@ -274,7 +277,7 @@ def is_connected(keys: str, **kw) -> bool:
|
|
274
277
|
return False
|
275
278
|
|
276
279
|
|
277
|
-
def make_connector(cls):
|
280
|
+
def make_connector(cls, _is_executor: bool = False):
|
278
281
|
"""
|
279
282
|
Register a class as a `Connector`.
|
280
283
|
The `type` will be the lower case of the class name, without the suffix `connector`.
|
@@ -300,7 +303,12 @@ def make_connector(cls):
|
|
300
303
|
>>>
|
301
304
|
"""
|
302
305
|
import re
|
303
|
-
|
306
|
+
suffix_regex = (
|
307
|
+
r'connector$'
|
308
|
+
if not _is_executor
|
309
|
+
else r'executor$'
|
310
|
+
)
|
311
|
+
typ = re.sub(suffix_regex, '', cls.__name__.lower())
|
304
312
|
with _locks['types']:
|
305
313
|
types[typ] = cls
|
306
314
|
with _locks['custom_types']:
|
@@ -363,3 +371,11 @@ def get_connector_plugin(
|
|
363
371
|
)
|
364
372
|
plugin = mrsm.Plugin(plugin_name)
|
365
373
|
return plugin if plugin.is_installed() else None
|
374
|
+
|
375
|
+
|
376
|
+
def _load_builtin_custom_connectors():
|
377
|
+
"""
|
378
|
+
Import custom connectors decorated with `@make_connector` or `@make_executor`.
|
379
|
+
"""
|
380
|
+
import meerschaum.jobs._SystemdExecutor
|
381
|
+
# import meerschaum.jobs._LocalExecutor
|