meerschaum 2.2.6__py3-none-any.whl → 2.3.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/__init__.py +4 -1
- meerschaum/__main__.py +10 -5
- meerschaum/_internal/arguments/_parser.py +44 -15
- meerschaum/_internal/entry.py +35 -14
- meerschaum/_internal/shell/Shell.py +155 -53
- meerschaum/_internal/shell/updates.py +175 -0
- meerschaum/actions/api.py +12 -12
- meerschaum/actions/attach.py +95 -0
- meerschaum/actions/delete.py +35 -26
- meerschaum/actions/register.py +19 -5
- meerschaum/actions/show.py +119 -148
- meerschaum/actions/start.py +85 -75
- meerschaum/actions/stop.py +68 -39
- meerschaum/actions/sync.py +3 -3
- meerschaum/actions/upgrade.py +28 -36
- meerschaum/api/_events.py +18 -1
- meerschaum/api/_oauth2.py +2 -0
- meerschaum/api/_websockets.py +2 -2
- meerschaum/api/dash/jobs.py +5 -2
- meerschaum/api/routes/__init__.py +1 -0
- meerschaum/api/routes/_actions.py +122 -44
- meerschaum/api/routes/_jobs.py +340 -0
- meerschaum/api/routes/_pipes.py +25 -25
- meerschaum/config/_default.py +1 -0
- meerschaum/config/_formatting.py +1 -0
- meerschaum/config/_paths.py +5 -0
- meerschaum/config/_shell.py +84 -67
- meerschaum/config/_version.py +1 -1
- meerschaum/config/static/__init__.py +9 -0
- meerschaum/connectors/__init__.py +9 -11
- meerschaum/connectors/api/APIConnector.py +18 -1
- meerschaum/connectors/api/_actions.py +60 -71
- meerschaum/connectors/api/_jobs.py +260 -0
- meerschaum/connectors/api/_misc.py +1 -1
- meerschaum/connectors/api/_request.py +13 -9
- meerschaum/connectors/parse.py +23 -7
- meerschaum/core/Pipe/_sync.py +3 -0
- meerschaum/plugins/__init__.py +89 -5
- meerschaum/utils/daemon/Daemon.py +333 -149
- meerschaum/utils/daemon/FileDescriptorInterceptor.py +19 -10
- meerschaum/utils/daemon/RotatingFile.py +18 -7
- meerschaum/utils/daemon/StdinFile.py +110 -0
- meerschaum/utils/daemon/__init__.py +40 -27
- meerschaum/utils/formatting/__init__.py +83 -37
- meerschaum/utils/formatting/_jobs.py +118 -51
- meerschaum/utils/formatting/_shell.py +6 -0
- meerschaum/utils/jobs/_Job.py +684 -0
- meerschaum/utils/jobs/__init__.py +245 -0
- meerschaum/utils/misc.py +18 -17
- meerschaum/utils/packages/__init__.py +21 -15
- meerschaum/utils/packages/_packages.py +2 -2
- meerschaum/utils/prompt.py +20 -7
- meerschaum/utils/schedule.py +21 -15
- {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dev1.dist-info}/METADATA +9 -9
- {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dev1.dist-info}/RECORD +61 -54
- {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dev1.dist-info}/WHEEL +1 -1
- {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dev1.dist-info}/LICENSE +0 -0
- {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dev1.dist-info}/NOTICE +0 -0
- {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dev1.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dev1.dist-info}/top_level.txt +0 -0
- {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dev1.dist-info}/zip-safe +0 -0
@@ -23,6 +23,8 @@ STATIC_CONFIG: Dict[str, Any] = {
|
|
23
23
|
'pipes': '/pipes',
|
24
24
|
'metadata': '/metadata',
|
25
25
|
'actions': '/actions',
|
26
|
+
'jobs': '/jobs',
|
27
|
+
'logs': '/logs',
|
26
28
|
'users': '/users',
|
27
29
|
'login': '/login',
|
28
30
|
'connectors': '/connectors',
|
@@ -39,6 +41,10 @@ STATIC_CONFIG: Dict[str, Any] = {
|
|
39
41
|
'token_expires_minutes': 720,
|
40
42
|
},
|
41
43
|
'webterm_job_name': '_webterm',
|
44
|
+
'default_timeout': 600,
|
45
|
+
'jobs': {
|
46
|
+
'stdin_message': 'MRSM_STDIN'
|
47
|
+
},
|
42
48
|
},
|
43
49
|
'sql': {
|
44
50
|
'internal_schema': '_mrsm_internal',
|
@@ -128,6 +134,9 @@ STATIC_CONFIG: Dict[str, Any] = {
|
|
128
134
|
},
|
129
135
|
'exists_timeout_seconds': 5.0,
|
130
136
|
},
|
137
|
+
'jobs': {
|
138
|
+
'check_restart_seconds': 1.0,
|
139
|
+
},
|
131
140
|
'setup': {
|
132
141
|
'name': 'meerschaum',
|
133
142
|
'formal_name': 'Meerschaum',
|
@@ -70,12 +70,12 @@ _loaded_plugin_connectors: bool = False
|
|
70
70
|
|
71
71
|
|
72
72
|
def get_connector(
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
73
|
+
type: str = None,
|
74
|
+
label: str = None,
|
75
|
+
refresh: bool = False,
|
76
|
+
debug: bool = False,
|
77
|
+
**kw: Any
|
78
|
+
) -> Connector:
|
79
79
|
"""
|
80
80
|
Return existing connector or create new connection and store for reuse.
|
81
81
|
|
@@ -274,9 +274,7 @@ def is_connected(keys: str, **kw) -> bool:
|
|
274
274
|
return False
|
275
275
|
|
276
276
|
|
277
|
-
def make_connector(
|
278
|
-
cls,
|
279
|
-
):
|
277
|
+
def make_connector(cls):
|
280
278
|
"""
|
281
279
|
Register a class as a `Connector`.
|
282
280
|
The `type` will be the lower case of the class name, without the suffix `connector`.
|
@@ -338,8 +336,8 @@ def load_plugin_connectors():
|
|
338
336
|
|
339
337
|
|
340
338
|
def get_connector_plugin(
|
341
|
-
|
342
|
-
|
339
|
+
connector: Connector,
|
340
|
+
) -> Union[str, None, mrsm.Plugin]:
|
343
341
|
"""
|
344
342
|
Determine the plugin for a connector.
|
345
343
|
This is useful for handling virtual environments for custom instance connectors.
|
@@ -33,7 +33,7 @@ class APIConnector(Connector):
|
|
33
33
|
delete,
|
34
34
|
wget,
|
35
35
|
)
|
36
|
-
from ._actions import get_actions, do_action
|
36
|
+
from ._actions import get_actions, do_action, do_action_async
|
37
37
|
from ._misc import get_mrsm_version, get_chaining_status
|
38
38
|
from ._pipes import (
|
39
39
|
register_pipe,
|
@@ -72,6 +72,23 @@ class APIConnector(Connector):
|
|
72
72
|
get_user_attributes,
|
73
73
|
)
|
74
74
|
from ._uri import from_uri
|
75
|
+
from ._jobs import (
|
76
|
+
get_jobs,
|
77
|
+
get_job,
|
78
|
+
get_job_metadata,
|
79
|
+
get_job_properties,
|
80
|
+
get_job_exists,
|
81
|
+
delete_job,
|
82
|
+
start_job,
|
83
|
+
create_job,
|
84
|
+
stop_job,
|
85
|
+
pause_job,
|
86
|
+
get_logs,
|
87
|
+
get_job_stop_time,
|
88
|
+
monitor_logs,
|
89
|
+
monitor_logs_async,
|
90
|
+
get_job_is_blocking_on_stdin,
|
91
|
+
)
|
75
92
|
|
76
93
|
def __init__(
|
77
94
|
self,
|
@@ -7,81 +7,70 @@ Functions to interact with /mrsm/actions
|
|
7
7
|
"""
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
|
-
from meerschaum.utils.typing import SuccessTuple, Optional, List
|
11
10
|
|
12
|
-
|
11
|
+
import json
|
12
|
+
import asyncio
|
13
|
+
from functools import partial
|
14
|
+
|
15
|
+
import meerschaum as mrsm
|
16
|
+
from meerschaum.utils.typing import SuccessTuple, List, Callable
|
17
|
+
from meerschaum.config.static import STATIC_CONFIG
|
18
|
+
|
19
|
+
ACTIONS_ENDPOINT: str = STATIC_CONFIG['api']['endpoints']['actions']
|
20
|
+
|
21
|
+
|
22
|
+
def get_actions(self):
|
13
23
|
"""Get available actions from the API instance."""
|
14
|
-
|
15
|
-
|
24
|
+
return self.get(ACTIONS_ENDPOINT)
|
25
|
+
|
26
|
+
|
27
|
+
def do_action(self, sysargs: List[str]) -> SuccessTuple:
|
28
|
+
"""
|
29
|
+
Execute a Meerschaum action remotely.
|
30
|
+
"""
|
31
|
+
return asyncio.run(self.do_action_async(sysargs))
|
16
32
|
|
17
33
|
|
18
|
-
def
|
34
|
+
async def do_action_async(
|
19
35
|
self,
|
20
|
-
|
21
|
-
|
22
|
-
debug: bool = False,
|
23
|
-
**kw
|
36
|
+
sysargs: List[str],
|
37
|
+
callback_function: Callable[[str], None] = partial(print, end=''),
|
24
38
|
) -> SuccessTuple:
|
25
|
-
"""Execute a Meerschaum action remotely.
|
26
|
-
|
27
|
-
If `sysargs` are provided, parse those instead.
|
28
|
-
Otherwise infer everything from keyword arguments.
|
29
|
-
|
30
|
-
Examples
|
31
|
-
--------
|
32
|
-
>>> conn = mrsm.get_connector('api:main')
|
33
|
-
>>> conn.do_action(['show', 'pipes'])
|
34
|
-
(True, "Success")
|
35
|
-
>>> conn.do_action(['show', 'arguments'], name='test')
|
36
|
-
(True, "Success")
|
37
39
|
"""
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
if
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
response_list = json.loads(response.text)
|
76
|
-
if isinstance(response_list, dict) and 'detail' in response_list:
|
77
|
-
return False, response_list['detail']
|
78
|
-
except Exception as e:
|
79
|
-
print(f"Invalid response: {response}")
|
80
|
-
print(e)
|
81
|
-
return False, response.text
|
82
|
-
if debug:
|
83
|
-
dprint(response)
|
84
|
-
try:
|
85
|
-
return response_list[0], response_list[1]
|
86
|
-
except Exception as e:
|
87
|
-
return False, f"Failed to parse result from action '{root_action}'"
|
40
|
+
Monitor a job's log files and await a callback with the changes.
|
41
|
+
"""
|
42
|
+
websockets, websockets_exceptions = mrsm.attempt_import('websockets', 'websockets.exceptions')
|
43
|
+
protocol = 'ws' if self.URI.startswith('http://') else 'wss'
|
44
|
+
port = self.port if 'port' in self.__dict__ else ''
|
45
|
+
uri = f"{protocol}://{self.host}:{port}{ACTIONS_ENDPOINT}/ws"
|
46
|
+
if sysargs and sysargs[0] == 'api' and len(sysargs) > 2:
|
47
|
+
sysargs = sysargs[2:]
|
48
|
+
|
49
|
+
sysargs_str = json.dumps(sysargs)
|
50
|
+
|
51
|
+
async with websockets.connect(uri) as websocket:
|
52
|
+
try:
|
53
|
+
await websocket.send(self.token or 'no-login')
|
54
|
+
response = await websocket.recv()
|
55
|
+
init_data = json.loads(response)
|
56
|
+
if not init_data.get('is_authenticated'):
|
57
|
+
return False, "Cannot authenticate with actions endpoint."
|
58
|
+
|
59
|
+
await websocket.send(sysargs_str)
|
60
|
+
except websockets_exceptions.ConnectionClosedOK:
|
61
|
+
return False, "Connection was closed."
|
62
|
+
|
63
|
+
while True:
|
64
|
+
try:
|
65
|
+
line = await websocket.recv()
|
66
|
+
if asyncio.iscoroutinefunction(callback_function):
|
67
|
+
await callback_function(line)
|
68
|
+
else:
|
69
|
+
callback_function(line)
|
70
|
+
except KeyboardInterrupt:
|
71
|
+
await websocket.close()
|
72
|
+
break
|
73
|
+
except websockets_exceptions.ConnectionClosedOK:
|
74
|
+
break
|
75
|
+
|
76
|
+
return True, "Success"
|
@@ -0,0 +1,260 @@
|
|
1
|
+
#! /usr/bin/env python3
|
2
|
+
# -*- coding: utf-8 -*-
|
3
|
+
# vim:fenc=utf-8
|
4
|
+
|
5
|
+
"""
|
6
|
+
Manage jobs via the Meerschaum API.
|
7
|
+
"""
|
8
|
+
|
9
|
+
import asyncio
|
10
|
+
from datetime import datetime
|
11
|
+
|
12
|
+
import meerschaum as mrsm
|
13
|
+
from meerschaum.utils.typing import Dict, Any, SuccessTuple, List, Union, Callable
|
14
|
+
from meerschaum.utils.jobs import Job
|
15
|
+
from meerschaum.config.static import STATIC_CONFIG
|
16
|
+
from meerschaum.utils.warnings import warn
|
17
|
+
|
18
|
+
JOBS_ENDPOINT: str = STATIC_CONFIG['api']['endpoints']['jobs']
|
19
|
+
LOGS_ENDPOINT: str = STATIC_CONFIG['api']['endpoints']['logs']
|
20
|
+
JOBS_STDIN_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stdin_message']
|
21
|
+
|
22
|
+
|
23
|
+
def get_jobs(self, debug: bool = False) -> Dict[str, Job]:
|
24
|
+
"""
|
25
|
+
Return a dictionary of remote jobs.
|
26
|
+
"""
|
27
|
+
response = self.get(JOBS_ENDPOINT, debug=debug)
|
28
|
+
if not response:
|
29
|
+
warn(f"Failed to get remote jobs from {self}.")
|
30
|
+
return {}
|
31
|
+
return {
|
32
|
+
name: Job(
|
33
|
+
name,
|
34
|
+
job_meta['sysargs'],
|
35
|
+
executor_keys=str(self),
|
36
|
+
_properties=job_meta['daemon']['properties']
|
37
|
+
)
|
38
|
+
for name, job_meta in response.json().items()
|
39
|
+
}
|
40
|
+
|
41
|
+
|
42
|
+
def get_job(self, name: str, debug: bool = False) -> Job:
|
43
|
+
"""
|
44
|
+
Return a single Job object.
|
45
|
+
"""
|
46
|
+
metadata = self.get_job_metadata(name, debug=debug)
|
47
|
+
if not metadata:
|
48
|
+
raise ValueError(f"Job '{name}' does not exist.")
|
49
|
+
|
50
|
+
return Job(
|
51
|
+
name,
|
52
|
+
metadata['sysargs'],
|
53
|
+
executor_keys=str(self),
|
54
|
+
_properties=metadata['daemon']['properties'],
|
55
|
+
)
|
56
|
+
|
57
|
+
|
58
|
+
def get_job_metadata(self, name: str, debug: bool = False) -> Dict[str, Any]:
|
59
|
+
"""
|
60
|
+
Return the metadata for a single job.
|
61
|
+
"""
|
62
|
+
response = self.get(JOBS_ENDPOINT + f"/{name}", debug=debug)
|
63
|
+
if not response:
|
64
|
+
if debug:
|
65
|
+
msg = (
|
66
|
+
response.json()['detail']
|
67
|
+
if 'detail' in response.text
|
68
|
+
else response.text
|
69
|
+
)
|
70
|
+
warn(f"Failed to get metadata for job '{name}':\n{msg}")
|
71
|
+
return {}
|
72
|
+
|
73
|
+
return response.json()
|
74
|
+
|
75
|
+
|
76
|
+
def get_job_properties(self, name: str, debug: bool = False) -> Dict[str, Any]:
|
77
|
+
"""
|
78
|
+
Return the daemon properties for a single job.
|
79
|
+
"""
|
80
|
+
metadata = self.get_job_metadata(name, debug=debug)
|
81
|
+
return metadata.get('daemon', {}).get('properties', {})
|
82
|
+
|
83
|
+
|
84
|
+
def get_job_exists(self, name: str, debug: bool = False) -> bool:
|
85
|
+
"""
|
86
|
+
Return whether a job exists.
|
87
|
+
"""
|
88
|
+
response = self.get(JOBS_ENDPOINT + f'/{name}/exists', debug=debug)
|
89
|
+
if not response:
|
90
|
+
warn(f"Failed to determine whether job '{name}' exists.")
|
91
|
+
return False
|
92
|
+
|
93
|
+
return response.json()
|
94
|
+
|
95
|
+
|
96
|
+
def delete_job(self, name: str, debug: bool = False) -> SuccessTuple:
|
97
|
+
"""
|
98
|
+
Delete a job.
|
99
|
+
"""
|
100
|
+
response = self.delete(JOBS_ENDPOINT + f"/{name}", debug=debug)
|
101
|
+
if not response:
|
102
|
+
if 'detail' in response.text:
|
103
|
+
return False, response.json()['detail']
|
104
|
+
|
105
|
+
return False, response.text
|
106
|
+
|
107
|
+
return tuple(response.json())
|
108
|
+
|
109
|
+
|
110
|
+
def start_job(self, name: str, debug: bool = False) -> SuccessTuple:
|
111
|
+
"""
|
112
|
+
Start a job.
|
113
|
+
"""
|
114
|
+
response = self.post(JOBS_ENDPOINT + f"/{name}/start", debug=debug)
|
115
|
+
if not response:
|
116
|
+
if 'detail' in response.text:
|
117
|
+
return False, response.json()['detail']
|
118
|
+
return False, response.text
|
119
|
+
|
120
|
+
return tuple(response.json())
|
121
|
+
|
122
|
+
|
123
|
+
def create_job(self, name: str, sysargs: List[str], debug: bool = False) -> SuccessTuple:
|
124
|
+
"""
|
125
|
+
Create a job.
|
126
|
+
"""
|
127
|
+
response = self.post(JOBS_ENDPOINT + f"/{name}", json=sysargs, debug=debug)
|
128
|
+
if not response:
|
129
|
+
if 'detail' in response.text:
|
130
|
+
return False, response.json()['detail']
|
131
|
+
return False, response.text
|
132
|
+
|
133
|
+
return tuple(response.json())
|
134
|
+
|
135
|
+
|
136
|
+
def stop_job(self, name: str, debug: bool = False) -> SuccessTuple:
|
137
|
+
"""
|
138
|
+
Stop a job.
|
139
|
+
"""
|
140
|
+
response = self.post(JOBS_ENDPOINT + f"/{name}/stop", debug=debug)
|
141
|
+
if not response:
|
142
|
+
if 'detail' in response.text:
|
143
|
+
return False, response.json()['detail']
|
144
|
+
return False, response.text
|
145
|
+
|
146
|
+
return tuple(response.json())
|
147
|
+
|
148
|
+
|
149
|
+
def pause_job(self, name: str, debug: bool = False) -> SuccessTuple:
|
150
|
+
"""
|
151
|
+
Pause a job.
|
152
|
+
"""
|
153
|
+
response = self.post(JOBS_ENDPOINT + f"/{name}/pause", debug=debug)
|
154
|
+
if not response:
|
155
|
+
if 'detail' in response.text:
|
156
|
+
return False, response.json()['detail']
|
157
|
+
return False, response.text
|
158
|
+
|
159
|
+
return tuple(response.json())
|
160
|
+
|
161
|
+
|
162
|
+
def get_logs(self, name: str, debug: bool = False) -> str:
|
163
|
+
"""
|
164
|
+
Return the logs for a job.
|
165
|
+
"""
|
166
|
+
response = self.get(LOGS_ENDPOINT + f"/{name}")
|
167
|
+
if not response:
|
168
|
+
raise ValueError(f"Cannot fetch logs for job '{name}':\n{response.text}")
|
169
|
+
|
170
|
+
return response.json()
|
171
|
+
|
172
|
+
|
173
|
+
def get_job_stop_time(self, name: str, debug: bool = False) -> Union[datetime, None]:
|
174
|
+
"""
|
175
|
+
Return the job's manual stop time.
|
176
|
+
"""
|
177
|
+
response = self.get(JOBS_ENDPOINT + f"/{name}/stop_time")
|
178
|
+
if not response:
|
179
|
+
warn(f"Failed to get stop time for job '{name}':\n{response.text}")
|
180
|
+
return None
|
181
|
+
|
182
|
+
data = response.json()
|
183
|
+
if data is None:
|
184
|
+
return None
|
185
|
+
|
186
|
+
return datetime.fromisoformat(data)
|
187
|
+
|
188
|
+
|
189
|
+
async def monitor_logs_async(
|
190
|
+
self,
|
191
|
+
name: str,
|
192
|
+
callback_function: Callable[[Any], Any],
|
193
|
+
input_callback_function: Callable[[], str],
|
194
|
+
accept_input: bool = True,
|
195
|
+
debug: bool = False,
|
196
|
+
):
|
197
|
+
"""
|
198
|
+
Monitor a job's log files and await a callback with the changes.
|
199
|
+
"""
|
200
|
+
websockets, websockets_exceptions = mrsm.attempt_import('websockets', 'websockets.exceptions')
|
201
|
+
protocol = 'ws' if self.URI.startswith('http://') else 'wss'
|
202
|
+
port = self.port if 'port' in self.__dict__ else ''
|
203
|
+
uri = f"{protocol}://{self.host}:{port}{LOGS_ENDPOINT}/{name}/ws"
|
204
|
+
|
205
|
+
async with websockets.connect(uri) as websocket:
|
206
|
+
try:
|
207
|
+
await websocket.send(self.token or 'no-login')
|
208
|
+
except websockets_exceptions.ConnectionClosedOK:
|
209
|
+
pass
|
210
|
+
|
211
|
+
while True:
|
212
|
+
try:
|
213
|
+
response = await websocket.recv()
|
214
|
+
if response == JOBS_STDIN_MESSAGE:
|
215
|
+
if asyncio.iscoroutinefunction(input_callback_function):
|
216
|
+
data = await input_callback_function()
|
217
|
+
else:
|
218
|
+
data = input_callback_function()
|
219
|
+
|
220
|
+
await websocket.send(data)
|
221
|
+
continue
|
222
|
+
|
223
|
+
if asyncio.iscoroutinefunction(callback_function):
|
224
|
+
await callback_function(response)
|
225
|
+
else:
|
226
|
+
callback_function(response)
|
227
|
+
except KeyboardInterrupt:
|
228
|
+
await websocket.close()
|
229
|
+
break
|
230
|
+
|
231
|
+
def monitor_logs(
|
232
|
+
self,
|
233
|
+
name: str,
|
234
|
+
callback_function: Callable[[Any], Any],
|
235
|
+
input_callback_function: Callable[[None], str],
|
236
|
+
accept_input: bool = True,
|
237
|
+
debug: bool = False,
|
238
|
+
):
|
239
|
+
"""
|
240
|
+
Monitor a job's log files and execute a callback with the changes.
|
241
|
+
"""
|
242
|
+
return asyncio.run(
|
243
|
+
self.monitor_logs_async(
|
244
|
+
name,
|
245
|
+
callback_function,
|
246
|
+
input_callback_function=input_callback_function,
|
247
|
+
accept_input=accept_input,
|
248
|
+
debug=debug
|
249
|
+
)
|
250
|
+
)
|
251
|
+
|
252
|
+
def get_job_is_blocking_on_stdin(self, name: str, debug: bool = False) -> bool:
|
253
|
+
"""
|
254
|
+
Return whether a remote job is blocking on stdin.
|
255
|
+
"""
|
256
|
+
response = self.get(JOBS_ENDPOINT + f'/{name}/is_blocking_on_stdin', debug=debug)
|
257
|
+
if not response:
|
258
|
+
return False
|
259
|
+
|
260
|
+
return response.json()
|
@@ -11,7 +11,7 @@ import urllib.parse
|
|
11
11
|
import pathlib
|
12
12
|
from meerschaum.utils.typing import Any, Optional, Dict, Union
|
13
13
|
from meerschaum.utils.debug import dprint
|
14
|
-
from meerschaum.
|
14
|
+
from meerschaum.config.static import STATIC_CONFIG
|
15
15
|
|
16
16
|
METHODS = {
|
17
17
|
'GET',
|
@@ -23,15 +23,16 @@ METHODS = {
|
|
23
23
|
'DELETE',
|
24
24
|
}
|
25
25
|
|
26
|
+
|
26
27
|
def make_request(
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
28
|
+
self,
|
29
|
+
method: str,
|
30
|
+
r_url: str,
|
31
|
+
headers: Optional[Dict[str, Any]] = None,
|
32
|
+
use_token: bool = True,
|
33
|
+
debug: bool = False,
|
34
|
+
**kwargs: Any
|
35
|
+
) -> 'requests.Response':
|
35
36
|
"""
|
36
37
|
Make a request to this APIConnector's endpoint using the in-memory session.
|
37
38
|
|
@@ -84,6 +85,9 @@ def make_request(
|
|
84
85
|
if use_token:
|
85
86
|
headers.update({'Authorization': f'Bearer {self.token}'})
|
86
87
|
|
88
|
+
if 'timeout' not in kwargs:
|
89
|
+
kwargs['timeout'] = STATIC_CONFIG['api']['default_timeout']
|
90
|
+
|
87
91
|
request_url = urllib.parse.urljoin(self.url, r_url)
|
88
92
|
if debug:
|
89
93
|
dprint(f"[{self}] Sending a '{method.upper()}' request to {request_url}")
|
meerschaum/connectors/parse.py
CHANGED
@@ -10,11 +10,11 @@ from __future__ import annotations
|
|
10
10
|
from meerschaum.utils.typing import Mapping, Any, SuccessTuple, Union, Optional, Dict, Tuple
|
11
11
|
|
12
12
|
def parse_connector_keys(
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
13
|
+
keys: str,
|
14
|
+
construct: bool = True,
|
15
|
+
as_tuple: bool = False,
|
16
|
+
**kw: Any
|
17
|
+
) -> (
|
18
18
|
Union[
|
19
19
|
meerschaum.connectors.Connector,
|
20
20
|
Dict[str, Any],
|
@@ -119,9 +119,25 @@ def parse_repo_keys(keys: Optional[str] = None, **kw):
|
|
119
119
|
return parse_connector_keys(keys, **kw)
|
120
120
|
|
121
121
|
|
122
|
+
def parse_executor_keys(keys: Optional[str] = None, **kw):
|
123
|
+
"""Parse the executor keys into an APIConnector or None."""
|
124
|
+
from meerschaum.config import get_config
|
125
|
+
if keys is None:
|
126
|
+
keys = get_config('meerschaum', 'default_executor')
|
127
|
+
|
128
|
+
if keys is None or keys == 'local':
|
129
|
+
return 'local'
|
130
|
+
|
131
|
+
keys = str(keys)
|
132
|
+
if ':' not in keys:
|
133
|
+
keys = 'api:' + keys
|
134
|
+
|
135
|
+
return parse_connector_keys(keys, **kw)
|
136
|
+
|
137
|
+
|
122
138
|
def is_valid_connector_keys(
|
123
|
-
|
124
|
-
|
139
|
+
keys: str
|
140
|
+
) -> bool:
|
125
141
|
"""Verify a connector_keys string references a valid connector.
|
126
142
|
"""
|
127
143
|
try:
|
meerschaum/core/Pipe/_sync.py
CHANGED
@@ -194,6 +194,9 @@ def sync(
|
|
194
194
|
if hasattr(df, 'MRSM_INFER_FETCH'):
|
195
195
|
try:
|
196
196
|
if p.connector is None:
|
197
|
+
if ':' not in p.connector_keys:
|
198
|
+
return True, f"{p} does not support fetching; nothing to do."
|
199
|
+
|
197
200
|
msg = f"{p} does not have a valid connector."
|
198
201
|
if p.connector_keys.startswith('plugin:'):
|
199
202
|
msg += f"\n Perhaps {p.connector_keys} has a syntax error?"
|