meerschaum 2.3.0.dev3__py3-none-any.whl → 2.3.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/__init__.py +3 -2
- meerschaum/__main__.py +0 -5
- meerschaum/_internal/arguments/_parser.py +6 -2
- meerschaum/_internal/entry.py +36 -6
- meerschaum/_internal/shell/Shell.py +32 -20
- meerschaum/actions/attach.py +12 -7
- meerschaum/actions/copy.py +68 -41
- meerschaum/actions/delete.py +64 -21
- meerschaum/actions/edit.py +3 -3
- meerschaum/actions/install.py +40 -32
- meerschaum/actions/pause.py +44 -27
- meerschaum/actions/restart.py +107 -0
- meerschaum/actions/show.py +8 -8
- meerschaum/actions/start.py +25 -40
- meerschaum/actions/stop.py +11 -4
- meerschaum/api/_events.py +10 -3
- meerschaum/api/dash/jobs.py +69 -70
- meerschaum/api/routes/_actions.py +8 -3
- meerschaum/api/routes/_jobs.py +37 -19
- meerschaum/config/_default.py +1 -1
- meerschaum/config/_paths.py +5 -0
- meerschaum/config/_version.py +1 -1
- meerschaum/config/static/__init__.py +3 -0
- meerschaum/connectors/Connector.py +13 -7
- meerschaum/connectors/__init__.py +21 -5
- meerschaum/connectors/api/APIConnector.py +3 -0
- meerschaum/connectors/api/_jobs.py +30 -3
- meerschaum/connectors/parse.py +10 -13
- meerschaum/core/Pipe/_bootstrap.py +16 -8
- meerschaum/jobs/_Executor.py +69 -0
- meerschaum/{utils/jobs → jobs}/_Job.py +160 -20
- meerschaum/jobs/_LocalExecutor.py +88 -0
- meerschaum/jobs/_SystemdExecutor.py +608 -0
- meerschaum/jobs/__init__.py +365 -0
- meerschaum/plugins/__init__.py +6 -6
- meerschaum/utils/daemon/Daemon.py +7 -0
- meerschaum/utils/daemon/RotatingFile.py +5 -2
- meerschaum/utils/daemon/StdinFile.py +12 -2
- meerschaum/utils/daemon/__init__.py +2 -0
- meerschaum/utils/formatting/_jobs.py +52 -16
- meerschaum/utils/misc.py +23 -5
- meerschaum/utils/packages/_packages.py +7 -4
- meerschaum/utils/process.py +9 -9
- meerschaum/utils/venv/__init__.py +2 -2
- {meerschaum-2.3.0.dev3.dist-info → meerschaum-2.3.0rc1.dist-info}/METADATA +14 -17
- {meerschaum-2.3.0.dev3.dist-info → meerschaum-2.3.0rc1.dist-info}/RECORD +52 -48
- meerschaum/utils/jobs/__init__.py +0 -245
- {meerschaum-2.3.0.dev3.dist-info → meerschaum-2.3.0rc1.dist-info}/LICENSE +0 -0
- {meerschaum-2.3.0.dev3.dist-info → meerschaum-2.3.0rc1.dist-info}/NOTICE +0 -0
- {meerschaum-2.3.0.dev3.dist-info → meerschaum-2.3.0rc1.dist-info}/WHEEL +0 -0
- {meerschaum-2.3.0.dev3.dist-info → meerschaum-2.3.0rc1.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.3.0.dev3.dist-info → meerschaum-2.3.0rc1.dist-info}/top_level.txt +0 -0
- {meerschaum-2.3.0.dev3.dist-info → meerschaum-2.3.0rc1.dist-info}/zip-safe +0 -0
@@ -13,7 +13,7 @@ from datetime import datetime
|
|
13
13
|
|
14
14
|
import meerschaum as mrsm
|
15
15
|
from meerschaum.utils.typing import Dict, Any, SuccessTuple, List, Union, Callable
|
16
|
-
from meerschaum.
|
16
|
+
from meerschaum.jobs import Job
|
17
17
|
from meerschaum.config.static import STATIC_CONFIG
|
18
18
|
from meerschaum.utils.warnings import warn, dprint
|
19
19
|
|
@@ -98,7 +98,6 @@ def get_job_metadata(self, name: str, debug: bool = False) -> Dict[str, Any]:
|
|
98
98
|
}
|
99
99
|
return metadata
|
100
100
|
|
101
|
-
|
102
101
|
def get_job_properties(self, name: str, debug: bool = False) -> Dict[str, Any]:
|
103
102
|
"""
|
104
103
|
Return the daemon properties for a single job.
|
@@ -106,6 +105,34 @@ def get_job_properties(self, name: str, debug: bool = False) -> Dict[str, Any]:
|
|
106
105
|
metadata = self.get_job_metadata(name, debug=debug)
|
107
106
|
return metadata.get('daemon', {}).get('properties', {})
|
108
107
|
|
108
|
+
def get_job_status(self, name: str, debug: bool = False) -> str:
|
109
|
+
"""
|
110
|
+
Return the job's status.
|
111
|
+
"""
|
112
|
+
metadata = self.get_job_metadata(name, debug=debug)
|
113
|
+
return metadata.get('status', 'stopped')
|
114
|
+
|
115
|
+
def get_job_began(self, name: str, debug: bool = False) -> str:
|
116
|
+
"""
|
117
|
+
Return a job's `began` timestamp, if it exists.
|
118
|
+
"""
|
119
|
+
properties = self.get_job_properties(name, debug=debug)
|
120
|
+
began_str = properties.get('daemon', {}).get('began', None)
|
121
|
+
if began_str is None:
|
122
|
+
return None
|
123
|
+
|
124
|
+
return began_str
|
125
|
+
|
126
|
+
def get_job_ended(self, name: str, debug: bool = False) -> str:
|
127
|
+
"""
|
128
|
+
Return a job's `ended` timestamp, if it exists.
|
129
|
+
"""
|
130
|
+
properties = self.get_job_properties(name, debug=debug)
|
131
|
+
ended_str = properties.get('daemon', {}).get('ended', None)
|
132
|
+
if ended_str is None:
|
133
|
+
return None
|
134
|
+
|
135
|
+
return ended_str
|
109
136
|
|
110
137
|
def get_job_exists(self, name: str, debug: bool = False) -> bool:
|
111
138
|
"""
|
@@ -226,7 +253,7 @@ async def monitor_logs_async(
|
|
226
253
|
"""
|
227
254
|
Monitor a job's log files and await a callback with the changes.
|
228
255
|
"""
|
229
|
-
from meerschaum.
|
256
|
+
from meerschaum.jobs import StopMonitoringLogs
|
230
257
|
from meerschaum.utils.formatting._jobs import strip_timestamp_from_line
|
231
258
|
|
232
259
|
websockets, websockets_exceptions = mrsm.attempt_import('websockets', 'websockets.exceptions')
|
meerschaum/connectors/parse.py
CHANGED
@@ -87,19 +87,17 @@ def parse_connector_keys(
|
|
87
87
|
|
88
88
|
|
89
89
|
def parse_instance_keys(
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
90
|
+
keys: Optional[str],
|
91
|
+
construct: bool = True,
|
92
|
+
as_tuple: bool = False,
|
93
|
+
**kw
|
94
|
+
):
|
95
95
|
"""
|
96
96
|
Parse the Meerschaum instance value into a Connector object.
|
97
97
|
"""
|
98
98
|
from meerschaum.utils.warnings import warn
|
99
99
|
from meerschaum.config import get_config
|
100
100
|
|
101
|
-
### TODO Check for valid types? Not sure how to do that if construct = False.
|
102
|
-
|
103
101
|
if keys is None:
|
104
102
|
keys = get_config('meerschaum', 'instance')
|
105
103
|
keys = str(keys)
|
@@ -120,25 +118,24 @@ def parse_repo_keys(keys: Optional[str] = None, **kw):
|
|
120
118
|
|
121
119
|
|
122
120
|
def parse_executor_keys(keys: Optional[str] = None, **kw):
|
123
|
-
"""Parse the executor keys into an APIConnector or
|
121
|
+
"""Parse the executor keys into an APIConnector or string."""
|
122
|
+
from meerschaum.jobs import get_executor_keys_from_context
|
124
123
|
from meerschaum.config import get_config
|
125
124
|
if keys is None:
|
126
|
-
keys =
|
125
|
+
keys = get_executor_keys_from_context()
|
127
126
|
|
128
127
|
if keys is None or keys == 'local':
|
129
128
|
return 'local'
|
130
129
|
|
131
130
|
keys = str(keys)
|
132
|
-
if ':' not in keys:
|
133
|
-
keys = 'api:' + keys
|
134
|
-
|
135
131
|
return parse_connector_keys(keys, **kw)
|
136
132
|
|
137
133
|
|
138
134
|
def is_valid_connector_keys(
|
139
135
|
keys: str
|
140
136
|
) -> bool:
|
141
|
-
"""
|
137
|
+
"""
|
138
|
+
Verify a connector_keys string references a valid connector.
|
142
139
|
"""
|
143
140
|
try:
|
144
141
|
success = parse_connector_keys(keys, construct=False) is not None
|
@@ -88,21 +88,29 @@ def bootstrap(
|
|
88
88
|
|
89
89
|
try:
|
90
90
|
if yes_no(
|
91
|
-
f"Would you like to edit the definition for {self}?",
|
91
|
+
f"Would you like to edit the definition for {self}?",
|
92
|
+
yes=yes,
|
93
|
+
noask=noask,
|
94
|
+
default='n',
|
92
95
|
):
|
93
96
|
edit_tuple = self.edit_definition(debug=debug)
|
94
97
|
if not edit_tuple[0]:
|
95
98
|
return edit_tuple
|
96
99
|
|
97
|
-
if yes_no(
|
100
|
+
if yes_no(
|
101
|
+
f"Would you like to try syncing {self} now?",
|
102
|
+
yes=yes,
|
103
|
+
noask=noask,
|
104
|
+
default='n',
|
105
|
+
):
|
98
106
|
sync_tuple = actions['sync'](
|
99
107
|
['pipes'],
|
100
|
-
connector_keys
|
101
|
-
metric_keys
|
102
|
-
location_keys
|
103
|
-
mrsm_instance
|
104
|
-
debug
|
105
|
-
shell
|
108
|
+
connector_keys=[self.connector_keys],
|
109
|
+
metric_keys=[self.metric_key],
|
110
|
+
location_keys=[self.location_key],
|
111
|
+
mrsm_instance=str(self.instance_connector),
|
112
|
+
debug=debug,
|
113
|
+
shell=shell,
|
106
114
|
)
|
107
115
|
if not sync_tuple[0]:
|
108
116
|
return sync_tuple
|
@@ -0,0 +1,69 @@
|
|
1
|
+
#! /usr/bin/env python3
|
2
|
+
# vim:fenc=utf-8
|
3
|
+
|
4
|
+
"""
|
5
|
+
Define the base class for a Job executor.
|
6
|
+
"""
|
7
|
+
|
8
|
+
from __future__ import annotations
|
9
|
+
|
10
|
+
from abc import abstractmethod
|
11
|
+
|
12
|
+
from meerschaum.connectors import Connector
|
13
|
+
from meerschaum.utils.typing import List, Dict, SuccessTuple, TYPE_CHECKING
|
14
|
+
|
15
|
+
if TYPE_CHECKING:
|
16
|
+
from meerschaum.jobs import Job
|
17
|
+
|
18
|
+
class Executor(Connector):
|
19
|
+
"""
|
20
|
+
Define the methods for managing jobs.
|
21
|
+
"""
|
22
|
+
|
23
|
+
@abstractmethod
|
24
|
+
def get_job_exists(self, name: str, debug: bool = False) -> bool:
|
25
|
+
"""
|
26
|
+
Return whether a job exists.
|
27
|
+
"""
|
28
|
+
|
29
|
+
@abstractmethod
|
30
|
+
def get_jobs(self) -> Dict[str, Job]:
|
31
|
+
"""
|
32
|
+
Return a dictionary of names -> Jobs.
|
33
|
+
"""
|
34
|
+
|
35
|
+
@abstractmethod
|
36
|
+
def create_job(self, name: str, sysargs: List[str], debug: bool = False) -> SuccessTuple:
|
37
|
+
"""
|
38
|
+
Create a new job.
|
39
|
+
"""
|
40
|
+
|
41
|
+
@abstractmethod
|
42
|
+
def start_job(self, name: str, debug: bool = False) -> SuccessTuple:
|
43
|
+
"""
|
44
|
+
Start a job.
|
45
|
+
"""
|
46
|
+
|
47
|
+
@abstractmethod
|
48
|
+
def stop_job(self, name: str, debug: bool = False) -> SuccessTuple:
|
49
|
+
"""
|
50
|
+
Stop a job.
|
51
|
+
"""
|
52
|
+
|
53
|
+
@abstractmethod
|
54
|
+
def pause_job(self, name: str, debug: bool = False) -> SuccessTuple:
|
55
|
+
"""
|
56
|
+
Pause a job.
|
57
|
+
"""
|
58
|
+
|
59
|
+
@abstractmethod
|
60
|
+
def delete_job(self, name: str, debug: bool = False) -> SuccessTuple:
|
61
|
+
"""
|
62
|
+
Delete a job.
|
63
|
+
"""
|
64
|
+
|
65
|
+
@abstractmethod
|
66
|
+
def get_logs(self, name: str, debug: bool = False) -> str:
|
67
|
+
"""
|
68
|
+
Return a job's log output.
|
69
|
+
"""
|
@@ -6,6 +6,8 @@
|
|
6
6
|
Define the Meerschaum abstraction atop daemons.
|
7
7
|
"""
|
8
8
|
|
9
|
+
from __future__ import annotations
|
10
|
+
|
9
11
|
import shlex
|
10
12
|
import asyncio
|
11
13
|
import threading
|
@@ -18,11 +20,17 @@ from functools import partial
|
|
18
20
|
from datetime import datetime, timezone
|
19
21
|
|
20
22
|
import meerschaum as mrsm
|
21
|
-
from meerschaum.utils.typing import
|
23
|
+
from meerschaum.utils.typing import (
|
24
|
+
List, Optional, Union, SuccessTuple, Any, Dict, Callable, TYPE_CHECKING,
|
25
|
+
)
|
22
26
|
from meerschaum._internal.entry import entry
|
23
27
|
from meerschaum.utils.warnings import warn
|
24
28
|
from meerschaum.config.paths import LOGS_RESOURCES_PATH
|
25
29
|
from meerschaum.config import get_config
|
30
|
+
from meerschaum.config.static import STATIC_CONFIG
|
31
|
+
|
32
|
+
if TYPE_CHECKING:
|
33
|
+
from meerschaum.jobs._Executor import Executor
|
26
34
|
|
27
35
|
BANNED_CHARS: List[str] = [
|
28
36
|
',', ';', "'", '"',
|
@@ -52,6 +60,10 @@ class Job:
|
|
52
60
|
sysargs: Union[List[str], str, None] = None,
|
53
61
|
executor_keys: Optional[str] = None,
|
54
62
|
_properties: Optional[Dict[str, Any]] = None,
|
63
|
+
_rotating_log = None,
|
64
|
+
_stdin_file = None,
|
65
|
+
_status_hook = None,
|
66
|
+
_result_hook = None,
|
55
67
|
):
|
56
68
|
"""
|
57
69
|
Create a new job to manage a `meerschaum.utils.daemon.Daemon`.
|
@@ -79,19 +91,40 @@ class Job:
|
|
79
91
|
if isinstance(sysargs, str):
|
80
92
|
sysargs = shlex.split(sysargs)
|
81
93
|
|
82
|
-
|
83
|
-
|
94
|
+
### NOTE: 'local' and 'systemd' executors are being coalesced.
|
95
|
+
if executor_keys is None:
|
96
|
+
from meerschaum.jobs import get_executor_keys_from_context
|
97
|
+
executor_keys = get_executor_keys_from_context()
|
98
|
+
|
84
99
|
self.executor_keys = executor_keys
|
85
100
|
self.name = name
|
86
101
|
try:
|
87
102
|
self._daemon = (
|
88
103
|
Daemon(daemon_id=name)
|
89
|
-
if executor_keys
|
104
|
+
if executor_keys == 'local'
|
90
105
|
else None
|
91
106
|
)
|
92
107
|
except Exception:
|
93
108
|
self._daemon = None
|
94
109
|
|
110
|
+
### Handle any injected dependencies.
|
111
|
+
if _rotating_log is not None:
|
112
|
+
self._rotating_log = _rotating_log
|
113
|
+
if self._daemon is not None:
|
114
|
+
self._daemon._rotating_log = _rotating_log
|
115
|
+
|
116
|
+
if _stdin_file is not None:
|
117
|
+
self._stdin_file = _stdin_file
|
118
|
+
if self._daemon is not None:
|
119
|
+
self._daemon._stdin_file = _stdin_file
|
120
|
+
self._daemon._blocking_stdin_file_path = _stdin_file.blocking_file_path
|
121
|
+
|
122
|
+
if _status_hook is not None:
|
123
|
+
self._status_hook = _status_hook
|
124
|
+
|
125
|
+
if _result_hook is not None:
|
126
|
+
self._result_hook = _result_hook
|
127
|
+
|
95
128
|
self._properties_patch = _properties or {}
|
96
129
|
|
97
130
|
daemon_sysargs = (
|
@@ -113,6 +146,71 @@ class Job:
|
|
113
146
|
self._properties_patch.update({'restart': True})
|
114
147
|
break
|
115
148
|
|
149
|
+
if '--systemd' in self._sysargs:
|
150
|
+
self._properties_patch.update({'systemd': True})
|
151
|
+
|
152
|
+
@staticmethod
|
153
|
+
def from_pid(pid: int, executor_keys: Optional[str] = None) -> Job:
|
154
|
+
"""
|
155
|
+
Build a `Job` from the PID of a running Meerschaum process.
|
156
|
+
|
157
|
+
Parameters
|
158
|
+
----------
|
159
|
+
pid: int
|
160
|
+
The PID of the process.
|
161
|
+
|
162
|
+
executor_keys: Optional[str], default None
|
163
|
+
The executor keys to assign to the job.
|
164
|
+
"""
|
165
|
+
from meerschaum.config.paths import DAEMON_RESOURCES_PATH
|
166
|
+
|
167
|
+
psutil = mrsm.attempt_import('psutil')
|
168
|
+
try:
|
169
|
+
process = psutil.Process(pid)
|
170
|
+
except psutil.NoSuchProcess as e:
|
171
|
+
warn(f"Process with PID {pid} does not exist.", stack=False)
|
172
|
+
raise e
|
173
|
+
|
174
|
+
command_args = process.cmdline()
|
175
|
+
is_daemon = command_args[1] == '-c'
|
176
|
+
|
177
|
+
if is_daemon:
|
178
|
+
daemon_id = command_args[-1].split('daemon_id=')[-1].split(')')[0].replace("'", '')
|
179
|
+
root_dir = process.environ().get(STATIC_CONFIG['environment']['root'], None)
|
180
|
+
if root_dir is None:
|
181
|
+
from meerschaum.config.paths import ROOT_DIR_PATH
|
182
|
+
root_dir = ROOT_DIR_PATH
|
183
|
+
jobs_dir = root_dir / DAEMON_RESOURCES_PATH.name
|
184
|
+
daemon_dir = jobs_dir / daemon_id
|
185
|
+
pid_file = daemon_dir / 'process.pid'
|
186
|
+
properties_path = daemon_dir / 'properties.json'
|
187
|
+
pickle_path = daemon_dir / 'pickle.pkl'
|
188
|
+
|
189
|
+
if pid_file.exists():
|
190
|
+
with open(pid_file, 'r', encoding='utf-8') as f:
|
191
|
+
daemon_pid = int(f.read())
|
192
|
+
|
193
|
+
if pid != daemon_pid:
|
194
|
+
raise EnvironmentError(f"Differing PIDs: {pid=}, {daemon_pid=}")
|
195
|
+
else:
|
196
|
+
raise EnvironmentError(f"Is job '{daemon_id}' running?")
|
197
|
+
|
198
|
+
return Job(daemon_id, executor_keys=executor_keys)
|
199
|
+
|
200
|
+
from meerschaum._internal.arguments._parse_arguments import parse_arguments
|
201
|
+
from meerschaum.utils.daemon import get_new_daemon_name
|
202
|
+
|
203
|
+
mrsm_ix = 0
|
204
|
+
for i, arg in enumerate(command_args):
|
205
|
+
if 'mrsm' in arg or 'meerschaum' in arg.lower():
|
206
|
+
mrsm_ix = i
|
207
|
+
break
|
208
|
+
|
209
|
+
sysargs = command_args[mrsm_ix+1:]
|
210
|
+
kwargs = parse_arguments(sysargs)
|
211
|
+
name = kwargs.get('name', get_new_daemon_name())
|
212
|
+
return Job(name, sysargs, executor_keys=executor_keys)
|
213
|
+
|
116
214
|
def start(self, debug: bool = False) -> SuccessTuple:
|
117
215
|
"""
|
118
216
|
Start the job's daemon.
|
@@ -144,6 +242,8 @@ class Job:
|
|
144
242
|
if self.daemon.status == 'stopped':
|
145
243
|
if not self.restart:
|
146
244
|
return True, f"{self} is not running."
|
245
|
+
elif self.stop_time is not None:
|
246
|
+
return True, f"{self} will not restart until manually started."
|
147
247
|
|
148
248
|
quit_success, quit_msg = self.daemon.quit(timeout=timeout_seconds)
|
149
249
|
if quit_success:
|
@@ -244,7 +344,7 @@ class Job:
|
|
244
344
|
|
245
345
|
stop_event: Optional[asyncio.Event], default None
|
246
346
|
If provided, stop monitoring when this event is set.
|
247
|
-
You may instead raise `meerschaum.
|
347
|
+
You may instead raise `meerschaum.jobs.StopMonitoringLogs`
|
248
348
|
from within `callback_function` to stop monitoring.
|
249
349
|
|
250
350
|
stop_on_exit: bool, default False
|
@@ -296,6 +396,9 @@ class Job:
|
|
296
396
|
stop_on_exit: bool = False,
|
297
397
|
strip_timestamps: bool = False,
|
298
398
|
accept_input: bool = True,
|
399
|
+
_logs_path: Optional[pathlib.Path] = None,
|
400
|
+
_log = None,
|
401
|
+
_stdin_file = None,
|
299
402
|
debug: bool = False,
|
300
403
|
):
|
301
404
|
"""
|
@@ -317,7 +420,7 @@ class Job:
|
|
317
420
|
|
318
421
|
stop_event: Optional[asyncio.Event], default None
|
319
422
|
If provided, stop monitoring when this event is set.
|
320
|
-
You may instead raise `meerschaum.
|
423
|
+
You may instead raise `meerschaum.jobs.StopMonitoringLogs`
|
321
424
|
from within `callback_function` to stop monitoring.
|
322
425
|
|
323
426
|
stop_on_exit: bool, default False
|
@@ -355,12 +458,14 @@ class Job:
|
|
355
458
|
}
|
356
459
|
combined_event = asyncio.Event()
|
357
460
|
emitted_text = False
|
461
|
+
stdin_file = _stdin_file if _stdin_file is not None else self.daemon.stdin_file
|
358
462
|
|
359
463
|
async def check_job_status():
|
360
464
|
nonlocal emitted_text
|
361
465
|
stopped_event = events.get('stopped', None)
|
362
466
|
if stopped_event is None:
|
363
467
|
return
|
468
|
+
|
364
469
|
sleep_time = 0.1
|
365
470
|
while sleep_time < 60:
|
366
471
|
if self.status == 'stopped':
|
@@ -408,7 +513,8 @@ class Job:
|
|
408
513
|
break
|
409
514
|
if not data.endswith('\n'):
|
410
515
|
data += '\n'
|
411
|
-
|
516
|
+
|
517
|
+
stdin_file.write(data)
|
412
518
|
await asyncio.sleep(0.1)
|
413
519
|
|
414
520
|
async def combine_events():
|
@@ -436,7 +542,7 @@ class Job:
|
|
436
542
|
check_blocking_on_input_task = asyncio.create_task(check_blocking_on_input())
|
437
543
|
combine_events_task = asyncio.create_task(combine_events())
|
438
544
|
|
439
|
-
log = self.daemon.rotating_log
|
545
|
+
log = _log if _log is not None else self.daemon.rotating_log
|
440
546
|
lines_to_show = get_config('jobs', 'logs', 'lines_to_show')
|
441
547
|
|
442
548
|
async def emit_latest_lines():
|
@@ -474,7 +580,7 @@ class Job:
|
|
474
580
|
|
475
581
|
watchfiles = mrsm.attempt_import('watchfiles')
|
476
582
|
async for changes in watchfiles.awatch(
|
477
|
-
LOGS_RESOURCES_PATH,
|
583
|
+
_logs_path or LOGS_RESOURCES_PATH,
|
478
584
|
stop_event=combined_event,
|
479
585
|
):
|
480
586
|
for change in changes:
|
@@ -485,7 +591,6 @@ class Job:
|
|
485
591
|
continue
|
486
592
|
|
487
593
|
await emit_latest_lines()
|
488
|
-
await emit_latest_lines()
|
489
594
|
|
490
595
|
await emit_latest_lines()
|
491
596
|
|
@@ -502,20 +607,16 @@ class Job:
|
|
502
607
|
"""
|
503
608
|
Write to a job's daemon's `stdin`.
|
504
609
|
"""
|
505
|
-
### TODO implement remote method?
|
506
|
-
if self.executor is not None:
|
507
|
-
pass
|
508
|
-
|
509
610
|
self.daemon.stdin_file.write(data)
|
510
611
|
|
511
612
|
@property
|
512
|
-
def executor(self) -> Union[
|
613
|
+
def executor(self) -> Union[Executor, None]:
|
513
614
|
"""
|
514
615
|
If the job is remote, return the connector to the remote API instance.
|
515
616
|
"""
|
516
617
|
return (
|
517
618
|
mrsm.get_connector(self.executor_keys)
|
518
|
-
if self.executor_keys
|
619
|
+
if self.executor_keys != 'local'
|
519
620
|
else None
|
520
621
|
)
|
521
622
|
|
@@ -524,10 +625,11 @@ class Job:
|
|
524
625
|
"""
|
525
626
|
Return the running status of the job's daemon.
|
526
627
|
"""
|
628
|
+
if '_status_hook' in self.__dict__:
|
629
|
+
return self._status_hook()
|
630
|
+
|
527
631
|
if self.executor is not None:
|
528
|
-
return self.executor.
|
529
|
-
self.name
|
530
|
-
).get('daemon', {}).get('status', 'stopped')
|
632
|
+
return self.executor.get_job_status(self.name)
|
531
633
|
|
532
634
|
return self.daemon.status
|
533
635
|
|
@@ -546,6 +648,9 @@ class Job:
|
|
546
648
|
"""
|
547
649
|
Return whether to restart a stopped job.
|
548
650
|
"""
|
651
|
+
if self.executor is not None:
|
652
|
+
return self.executor.get_job_metadata(self.name).get('restart', False)
|
653
|
+
|
549
654
|
return self.daemon.properties.get('restart', False)
|
550
655
|
|
551
656
|
@property
|
@@ -556,6 +661,15 @@ class Job:
|
|
556
661
|
if self.is_running():
|
557
662
|
return True, f"{self} is running."
|
558
663
|
|
664
|
+
if '_result_hook' in self.__dict__:
|
665
|
+
return self._result_hook()
|
666
|
+
|
667
|
+
if self.executor is not None:
|
668
|
+
return (
|
669
|
+
self.executor.get_job_metadata(self.name)
|
670
|
+
.get('result', (False, "No result available."))
|
671
|
+
)
|
672
|
+
|
559
673
|
_result = self.daemon.properties.get('result', None)
|
560
674
|
if _result is None:
|
561
675
|
return False, "No result available."
|
@@ -570,7 +684,9 @@ class Job:
|
|
570
684
|
if self._sysargs:
|
571
685
|
return self._sysargs
|
572
686
|
|
573
|
-
|
687
|
+
if self.executor is not None:
|
688
|
+
return self.executor.get_job_metadata(self.name).get('sysargs', [])
|
689
|
+
|
574
690
|
target_args = self.daemon.target_args
|
575
691
|
if target_args is None:
|
576
692
|
return []
|
@@ -601,6 +717,12 @@ class Job:
|
|
601
717
|
label=shlex.join(self._sysargs),
|
602
718
|
properties=properties,
|
603
719
|
)
|
720
|
+
if '_rotating_log' in self.__dict__:
|
721
|
+
self._daemon._rotating_log = self._rotating_log
|
722
|
+
|
723
|
+
if '_stdin_file' in self.__dict__:
|
724
|
+
self._daemon._stdin_file = self._stdin_file
|
725
|
+
self._daemon._blocking_stdin_file_path = self._stdin_file.blocking_file_path
|
604
726
|
|
605
727
|
return self._daemon
|
606
728
|
|
@@ -609,6 +731,16 @@ class Job:
|
|
609
731
|
"""
|
610
732
|
The datetime when the job began running.
|
611
733
|
"""
|
734
|
+
if self.executor is not None:
|
735
|
+
began_str = self.executor.get_job_began(name)
|
736
|
+
if began_str is None:
|
737
|
+
return None
|
738
|
+
return (
|
739
|
+
datetime.fromisoformat(began_str)
|
740
|
+
.astimezone(timezone.utc)
|
741
|
+
.replace(tzinfo=None)
|
742
|
+
)
|
743
|
+
|
612
744
|
began_str = self.daemon.properties.get('process', {}).get('began', None)
|
613
745
|
if began_str is None:
|
614
746
|
return None
|
@@ -620,6 +752,14 @@ class Job:
|
|
620
752
|
"""
|
621
753
|
The datetime when the job stopped running.
|
622
754
|
"""
|
755
|
+
if self.executor is not None:
|
756
|
+
ended_str = self.executor.get_job_ended(self.name)
|
757
|
+
return (
|
758
|
+
datetime.fromisoformat(ended_str)
|
759
|
+
.astimezone(timezone.utc)
|
760
|
+
.replace(tzinfo=None)
|
761
|
+
)
|
762
|
+
|
623
763
|
ended_str = self.daemon.properties.get('process', {}).get('ended', None)
|
624
764
|
if ended_str is None:
|
625
765
|
return None
|
@@ -0,0 +1,88 @@
|
|
1
|
+
#! /usr/bin/env python3
|
2
|
+
# vim:fenc=utf-8
|
3
|
+
|
4
|
+
"""
|
5
|
+
Run jobs locally.
|
6
|
+
"""
|
7
|
+
|
8
|
+
from meerschaum.utils.typing import Dict, Any, List, SuccessTuple, Union
|
9
|
+
from meerschaum.jobs import Job, Executor, make_executor
|
10
|
+
from meerschaum.utils.daemon import Daemon, get_daemons
|
11
|
+
from meerschaum._internal.entry import entry
|
12
|
+
|
13
|
+
|
14
|
+
@make_executor
|
15
|
+
class LocalExecutor(Executor):
|
16
|
+
"""
|
17
|
+
Run jobs locally as Unix daemons.
|
18
|
+
"""
|
19
|
+
|
20
|
+
def get_job_daemon(
|
21
|
+
self,
|
22
|
+
name: str,
|
23
|
+
# sysargs: Opt
|
24
|
+
debug: bool = False,
|
25
|
+
) -> Union[Daemon, None]:
|
26
|
+
"""
|
27
|
+
Return a job's daemon if it exists.
|
28
|
+
"""
|
29
|
+
try:
|
30
|
+
daemon = Daemon(name)
|
31
|
+
except Exception:
|
32
|
+
daemon = None
|
33
|
+
|
34
|
+
return daemon
|
35
|
+
|
36
|
+
def get_daemon_syargs(self, name: str, debug: bool = False) -> Union[List[str], None]:
|
37
|
+
"""
|
38
|
+
Return the list of sysargs from the job's daemon.
|
39
|
+
"""
|
40
|
+
daemon = self.get_job_daemon(name, debug=debug)
|
41
|
+
|
42
|
+
if daemon is None:
|
43
|
+
return None
|
44
|
+
|
45
|
+
return daemon.properties.get('target', {}).get('args', [None])[0]
|
46
|
+
|
47
|
+
def get_job_exists(self, name: str, debug: bool = False) -> bool:
|
48
|
+
"""
|
49
|
+
Return whether a job exists.
|
50
|
+
"""
|
51
|
+
daemon = self.get_job_daemon(name, debug=debug)
|
52
|
+
if daemon is None:
|
53
|
+
return False
|
54
|
+
|
55
|
+
def get_jobs(self) -> Dict[str, Job]:
|
56
|
+
"""
|
57
|
+
Return a dictionary of names -> Jobs.
|
58
|
+
"""
|
59
|
+
|
60
|
+
def create_job(self, name: str, sysargs: List[str], debug: bool = False) -> SuccessTuple:
|
61
|
+
"""
|
62
|
+
Create a new job.
|
63
|
+
"""
|
64
|
+
|
65
|
+
def start_job(self, name: str, debug: bool = False) -> SuccessTuple:
|
66
|
+
"""
|
67
|
+
Start a job.
|
68
|
+
"""
|
69
|
+
|
70
|
+
def stop_job(self, name: str, debug: bool = False) -> SuccessTuple:
|
71
|
+
"""
|
72
|
+
Stop a job.
|
73
|
+
"""
|
74
|
+
|
75
|
+
def pause_job(self, name: str, debug: bool = False) -> SuccessTuple:
|
76
|
+
"""
|
77
|
+
Pause a job.
|
78
|
+
"""
|
79
|
+
|
80
|
+
def delete_job(self, name: str, debug: bool = False) -> SuccessTuple:
|
81
|
+
"""
|
82
|
+
Delete a job.
|
83
|
+
"""
|
84
|
+
|
85
|
+
def get_logs(self, name: str, debug: bool = False) -> str:
|
86
|
+
"""
|
87
|
+
Return a job's log output.
|
88
|
+
"""
|