meerschaum 2.2.7__py3-none-any.whl → 2.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/__init__.py +6 -1
- meerschaum/__main__.py +0 -5
- meerschaum/_internal/arguments/__init__.py +1 -1
- meerschaum/_internal/arguments/_parse_arguments.py +72 -6
- meerschaum/_internal/arguments/_parser.py +45 -15
- meerschaum/_internal/docs/index.py +265 -8
- meerschaum/_internal/entry.py +154 -24
- meerschaum/_internal/shell/Shell.py +264 -77
- meerschaum/actions/__init__.py +29 -17
- meerschaum/actions/api.py +12 -12
- meerschaum/actions/attach.py +113 -0
- meerschaum/actions/copy.py +68 -41
- meerschaum/actions/delete.py +112 -50
- meerschaum/actions/edit.py +3 -3
- meerschaum/actions/install.py +40 -32
- meerschaum/actions/pause.py +44 -27
- meerschaum/actions/restart.py +107 -0
- meerschaum/actions/show.py +130 -159
- meerschaum/actions/start.py +161 -100
- meerschaum/actions/stop.py +78 -42
- meerschaum/api/_events.py +25 -1
- meerschaum/api/_oauth2.py +2 -0
- meerschaum/api/_websockets.py +2 -2
- meerschaum/api/dash/callbacks/jobs.py +36 -44
- meerschaum/api/dash/jobs.py +89 -78
- meerschaum/api/routes/__init__.py +1 -0
- meerschaum/api/routes/_actions.py +148 -17
- meerschaum/api/routes/_jobs.py +407 -0
- meerschaum/api/routes/_pipes.py +5 -5
- meerschaum/config/_default.py +1 -0
- meerschaum/config/_jobs.py +1 -1
- meerschaum/config/_paths.py +7 -0
- meerschaum/config/_shell.py +8 -3
- meerschaum/config/_version.py +1 -1
- meerschaum/config/static/__init__.py +17 -0
- meerschaum/connectors/Connector.py +13 -7
- meerschaum/connectors/__init__.py +28 -15
- meerschaum/connectors/api/APIConnector.py +27 -1
- meerschaum/connectors/api/_actions.py +71 -6
- meerschaum/connectors/api/_jobs.py +368 -0
- meerschaum/connectors/api/_pipes.py +85 -84
- meerschaum/connectors/parse.py +27 -15
- meerschaum/core/Pipe/_bootstrap.py +16 -8
- meerschaum/jobs/_Executor.py +69 -0
- meerschaum/jobs/_Job.py +899 -0
- meerschaum/jobs/__init__.py +396 -0
- meerschaum/jobs/systemd.py +694 -0
- meerschaum/plugins/__init__.py +97 -12
- meerschaum/utils/daemon/Daemon.py +276 -30
- meerschaum/utils/daemon/FileDescriptorInterceptor.py +5 -5
- meerschaum/utils/daemon/RotatingFile.py +14 -7
- meerschaum/utils/daemon/StdinFile.py +121 -0
- meerschaum/utils/daemon/__init__.py +15 -7
- meerschaum/utils/daemon/_names.py +15 -13
- meerschaum/utils/formatting/__init__.py +2 -1
- meerschaum/utils/formatting/_jobs.py +115 -62
- meerschaum/utils/formatting/_shell.py +6 -0
- meerschaum/utils/misc.py +41 -22
- meerschaum/utils/packages/_packages.py +9 -6
- meerschaum/utils/process.py +9 -9
- meerschaum/utils/prompt.py +16 -8
- meerschaum/utils/venv/__init__.py +2 -2
- {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dist-info}/METADATA +22 -25
- {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dist-info}/RECORD +70 -61
- {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dist-info}/WHEEL +1 -1
- {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dist-info}/LICENSE +0 -0
- {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dist-info}/NOTICE +0 -0
- {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dist-info}/top_level.txt +0 -0
- {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dist-info}/zip-safe +0 -0
meerschaum/plugins/__init__.py
CHANGED
@@ -8,6 +8,7 @@ Expose plugin management APIs from the `meerschaum.plugins` module.
|
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
10
|
import functools
|
11
|
+
import meerschaum as mrsm
|
11
12
|
from meerschaum.utils.typing import Callable, Any, Union, Optional, Dict, List, Tuple
|
12
13
|
from meerschaum.utils.threading import Lock, RLock
|
13
14
|
from meerschaum.plugins._Plugin import Plugin
|
@@ -27,7 +28,8 @@ _locks = {
|
|
27
28
|
'PLUGINS_INTERNAL_LOCK_PATH': RLock(),
|
28
29
|
}
|
29
30
|
__all__ = (
|
30
|
-
"Plugin", "make_action", "api_plugin", "dash_plugin", "
|
31
|
+
"Plugin", "make_action", "api_plugin", "dash_plugin", "web_page",
|
32
|
+
"import_plugins", "from_plugin_import",
|
31
33
|
"reload_plugins", "get_plugins", "get_data_plugins", "add_plugin_argument",
|
32
34
|
"pre_sync_hook", "post_sync_hook",
|
33
35
|
)
|
@@ -36,12 +38,12 @@ __pdoc__ = {
|
|
36
38
|
}
|
37
39
|
|
38
40
|
def make_action(
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
41
|
+
function: Callable[[Any], Any],
|
42
|
+
shell: bool = False,
|
43
|
+
activate: bool = True,
|
44
|
+
deactivate: bool = True,
|
45
|
+
debug: bool = False
|
46
|
+
) -> Callable[[Any], Any]:
|
45
47
|
"""
|
46
48
|
Make a function a Meerschaum action. Useful for plugins that are adding multiple actions.
|
47
49
|
|
@@ -429,11 +431,11 @@ def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
|
|
429
431
|
|
430
432
|
|
431
433
|
def import_plugins(
|
432
|
-
|
433
|
-
|
434
|
-
|
435
|
-
|
436
|
-
|
434
|
+
*plugins_to_import: Union[str, List[str], None],
|
435
|
+
warn: bool = True,
|
436
|
+
) -> Union[
|
437
|
+
'ModuleType', Tuple['ModuleType', None]
|
438
|
+
]:
|
437
439
|
"""
|
438
440
|
Import the Meerschaum plugins directory.
|
439
441
|
|
@@ -524,6 +526,89 @@ def import_plugins(
|
|
524
526
|
return imported_plugins
|
525
527
|
|
526
528
|
|
529
|
+
def from_plugin_import(plugin_import_name: str, *attrs: str) -> Any:
|
530
|
+
"""
|
531
|
+
Emulate the `from module import x` behavior.
|
532
|
+
|
533
|
+
Parameters
|
534
|
+
----------
|
535
|
+
plugin_import_name: str
|
536
|
+
The import name of the plugin's module.
|
537
|
+
Separate submodules with '.' (e.g. 'compose.utils.pipes')
|
538
|
+
|
539
|
+
attrs: str
|
540
|
+
Names of the attributes to return.
|
541
|
+
|
542
|
+
Returns
|
543
|
+
-------
|
544
|
+
Objects from a plugin's submodule.
|
545
|
+
If multiple objects are provided, return a tuple.
|
546
|
+
|
547
|
+
Examples
|
548
|
+
--------
|
549
|
+
>>> init = from_plugin_import('compose.utils', 'init')
|
550
|
+
>>> with mrsm.Venv('compose'):
|
551
|
+
... cf = init()
|
552
|
+
>>> build_parent_pipe, get_defined_pipes = from_plugin_import(
|
553
|
+
... 'compose.utils.pipes',
|
554
|
+
... 'build_parent_pipe',
|
555
|
+
... 'get_defined_pipes',
|
556
|
+
... )
|
557
|
+
>>> parent_pipe = build_parent_pipe(cf)
|
558
|
+
>>> defined_pipes = get_defined_pipes(cf)
|
559
|
+
"""
|
560
|
+
import importlib
|
561
|
+
from meerschaum.config._paths import PLUGINS_RESOURCES_PATH
|
562
|
+
from meerschaum.utils.warnings import warn as _warn
|
563
|
+
if plugin_import_name.startswith('plugins.'):
|
564
|
+
plugin_import_name = plugin_import_name[len('plugins.'):]
|
565
|
+
plugin_import_parts = plugin_import_name.split('.')
|
566
|
+
plugin_root_name = plugin_import_parts[0]
|
567
|
+
plugin = mrsm.Plugin(plugin_root_name)
|
568
|
+
|
569
|
+
submodule_import_name = '.'.join(
|
570
|
+
[PLUGINS_RESOURCES_PATH.stem]
|
571
|
+
+ plugin_import_parts
|
572
|
+
)
|
573
|
+
if len(attrs) == 0:
|
574
|
+
raise ValueError(f"Provide which attributes to return from '{submodule_import_name}'.")
|
575
|
+
|
576
|
+
attrs_to_return = []
|
577
|
+
with mrsm.Venv(plugin):
|
578
|
+
if plugin.module is None:
|
579
|
+
return None
|
580
|
+
|
581
|
+
try:
|
582
|
+
submodule = importlib.import_module(submodule_import_name)
|
583
|
+
except ImportError as e:
|
584
|
+
_warn(
|
585
|
+
f"Failed to import plugin '{submodule_import_name}':\n "
|
586
|
+
+ f"{e}\n\nHere's a stacktrace:",
|
587
|
+
stack=False,
|
588
|
+
)
|
589
|
+
from meerschaum.utils.formatting import get_console
|
590
|
+
get_console().print_exception(
|
591
|
+
suppress=[
|
592
|
+
'meerschaum/plugins/__init__.py',
|
593
|
+
importlib,
|
594
|
+
importlib._bootstrap,
|
595
|
+
]
|
596
|
+
)
|
597
|
+
return None
|
598
|
+
|
599
|
+
for attr in attrs:
|
600
|
+
try:
|
601
|
+
attrs_to_return.append(getattr(submodule, attr))
|
602
|
+
except Exception:
|
603
|
+
_warn(f"Failed to access '{attr}' from '{submodule_import_name}'.")
|
604
|
+
attrs_to_return.append(None)
|
605
|
+
|
606
|
+
if len(attrs) == 1:
|
607
|
+
return attrs_to_return[0]
|
608
|
+
|
609
|
+
return tuple(attrs_to_return)
|
610
|
+
|
611
|
+
|
527
612
|
def load_plugins(debug: bool = False, shell: bool = False) -> None:
|
528
613
|
"""
|
529
614
|
Import Meerschaum plugins and update the actions dictionary.
|
@@ -8,6 +8,7 @@ Manage running daemons via the Daemon class.
|
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
10
|
import os
|
11
|
+
import importlib
|
11
12
|
import pathlib
|
12
13
|
import json
|
13
14
|
import shutil
|
@@ -21,7 +22,7 @@ from datetime import datetime, timezone
|
|
21
22
|
import meerschaum as mrsm
|
22
23
|
from meerschaum.utils.typing import (
|
23
24
|
Optional, Dict, Any, SuccessTuple, Callable, List, Union,
|
24
|
-
is_success_tuple,
|
25
|
+
is_success_tuple, Tuple,
|
25
26
|
)
|
26
27
|
from meerschaum.config import get_config
|
27
28
|
from meerschaum.config.static import STATIC_CONFIG
|
@@ -34,6 +35,7 @@ from meerschaum.utils.packages import attempt_import
|
|
34
35
|
from meerschaum.utils.venv import venv_exec
|
35
36
|
from meerschaum.utils.daemon._names import get_new_daemon_name
|
36
37
|
from meerschaum.utils.daemon.RotatingFile import RotatingFile
|
38
|
+
from meerschaum.utils.daemon.StdinFile import StdinFile
|
37
39
|
from meerschaum.utils.threading import RepeatTimer
|
38
40
|
from meerschaum.__main__ import _close_pools
|
39
41
|
|
@@ -43,6 +45,32 @@ _results = {}
|
|
43
45
|
class Daemon:
|
44
46
|
"""
|
45
47
|
Daemonize Python functions into background processes.
|
48
|
+
|
49
|
+
Examples
|
50
|
+
--------
|
51
|
+
>>> import meerschaum as mrsm
|
52
|
+
>>> from meerschaum.utils.daemons import Daemon
|
53
|
+
>>> daemon = Daemon(print, ('hi',))
|
54
|
+
>>> success, msg = daemon.run()
|
55
|
+
>>> print(daemon.log_text)
|
56
|
+
|
57
|
+
2024-07-29 18:03 | hi
|
58
|
+
2024-07-29 18:03 |
|
59
|
+
>>> daemon.run(allow_dirty_run=True)
|
60
|
+
>>> print(daemon.log_text)
|
61
|
+
|
62
|
+
2024-07-29 18:03 | hi
|
63
|
+
2024-07-29 18:03 |
|
64
|
+
2024-07-29 18:05 | hi
|
65
|
+
2024-07-29 18:05 |
|
66
|
+
>>> mrsm.pprint(daemon.properties)
|
67
|
+
{
|
68
|
+
'label': 'print',
|
69
|
+
'target': {'name': 'print', 'module': 'builtins', 'args': ['hi'], 'kw': {}},
|
70
|
+
'result': None,
|
71
|
+
'process': {'ended': '2024-07-29T18:03:33.752806'}
|
72
|
+
}
|
73
|
+
|
46
74
|
"""
|
47
75
|
|
48
76
|
def __new__(
|
@@ -61,10 +89,57 @@ class Daemon:
|
|
61
89
|
instance = instance.read_pickle()
|
62
90
|
return instance
|
63
91
|
|
92
|
+
@classmethod
|
93
|
+
def from_properties_file(cls, daemon_id: str) -> Daemon:
|
94
|
+
"""
|
95
|
+
Return a Daemon from a properties dictionary.
|
96
|
+
"""
|
97
|
+
properties_path = cls._get_properties_path_from_daemon_id(daemon_id)
|
98
|
+
if not properties_path.exists():
|
99
|
+
raise OSError(f"Properties file '{properties_path}' does not exist.")
|
100
|
+
|
101
|
+
try:
|
102
|
+
with open(properties_path, 'r', encoding='utf-8') as f:
|
103
|
+
properties = json.load(f)
|
104
|
+
except Exception:
|
105
|
+
properties = {}
|
106
|
+
|
107
|
+
if not properties:
|
108
|
+
raise ValueError(f"No properties could be read for daemon '{daemon_id}'.")
|
109
|
+
|
110
|
+
daemon_id = properties_path.parent.name
|
111
|
+
target_cf = properties.get('target', {})
|
112
|
+
target_module_name = target_cf.get('module', None)
|
113
|
+
target_function_name = target_cf.get('name', None)
|
114
|
+
target_args = target_cf.get('args', None)
|
115
|
+
target_kw = target_cf.get('kw', None)
|
116
|
+
label = properties.get('label', None)
|
117
|
+
|
118
|
+
if None in [
|
119
|
+
target_module_name,
|
120
|
+
target_function_name,
|
121
|
+
target_args,
|
122
|
+
target_kw,
|
123
|
+
]:
|
124
|
+
raise ValueError("Missing target function information.")
|
125
|
+
|
126
|
+
target_module = importlib.import_module(target_module_name)
|
127
|
+
target_function = getattr(target_module, target_function_name)
|
128
|
+
|
129
|
+
return Daemon(
|
130
|
+
daemon_id=daemon_id,
|
131
|
+
target=target_function,
|
132
|
+
target_args=target_args,
|
133
|
+
target_kw=target_kw,
|
134
|
+
properties=properties,
|
135
|
+
label=label,
|
136
|
+
)
|
137
|
+
|
138
|
+
|
64
139
|
def __init__(
|
65
140
|
self,
|
66
141
|
target: Optional[Callable[[Any], Any]] = None,
|
67
|
-
target_args:
|
142
|
+
target_args: Union[List[Any], Tuple[Any], None] = None,
|
68
143
|
target_kw: Optional[Dict[str, Any]] = None,
|
69
144
|
daemon_id: Optional[str] = None,
|
70
145
|
label: Optional[str] = None,
|
@@ -76,7 +151,7 @@ class Daemon:
|
|
76
151
|
target: Optional[Callable[[Any], Any]], default None,
|
77
152
|
The function to execute in a child process.
|
78
153
|
|
79
|
-
target_args:
|
154
|
+
target_args: Union[List[Any], Tuple[Any], None], default None
|
80
155
|
Positional arguments to pass to the target function.
|
81
156
|
|
82
157
|
target_kw: Optional[Dict[str, Any]], default None
|
@@ -91,25 +166,54 @@ class Daemon:
|
|
91
166
|
Label string to help identifiy a daemon.
|
92
167
|
If `None`, use the function name instead.
|
93
168
|
|
94
|
-
|
169
|
+
properties: Optional[Dict[str, Any]], default None
|
95
170
|
Override reading from the properties JSON by providing an existing dictionary.
|
96
171
|
"""
|
97
172
|
_pickle = self.__dict__.get('_pickle', False)
|
98
173
|
if daemon_id is not None:
|
99
174
|
self.daemon_id = daemon_id
|
100
175
|
if not self.pickle_path.exists() and not target and ('target' not in self.__dict__):
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
176
|
+
|
177
|
+
if not self.properties_path.exists():
|
178
|
+
raise Exception(
|
179
|
+
f"Daemon '{self.daemon_id}' does not exist. "
|
180
|
+
+ "Pass a target to create a new Daemon."
|
181
|
+
)
|
182
|
+
|
183
|
+
try:
|
184
|
+
new_daemon = self.from_properties_file(daemon_id)
|
185
|
+
except Exception:
|
186
|
+
new_daemon = None
|
187
|
+
|
188
|
+
if new_daemon is not None:
|
189
|
+
new_daemon.write_pickle()
|
190
|
+
target = new_daemon.target
|
191
|
+
target_args = new_daemon.target_args
|
192
|
+
target_kw = new_daemon.target_kw
|
193
|
+
label = new_daemon.label
|
194
|
+
self._properties = new_daemon.properties
|
195
|
+
else:
|
196
|
+
try:
|
197
|
+
self.properties_path.unlink()
|
198
|
+
except Exception:
|
199
|
+
pass
|
200
|
+
|
201
|
+
raise Exception(
|
202
|
+
f"Could not recover daemon '{self.daemon_id}' "
|
203
|
+
+ "from its properties file."
|
204
|
+
)
|
205
|
+
|
105
206
|
if 'target' not in self.__dict__:
|
106
207
|
if target is None:
|
107
208
|
error("Cannot create a Daemon without a target.")
|
108
209
|
self.target = target
|
109
|
-
|
110
|
-
|
111
|
-
if '
|
112
|
-
self.
|
210
|
+
|
211
|
+
### NOTE: We have to check self.__dict__ in case we un-pickling.
|
212
|
+
if '_target_args' not in self.__dict__:
|
213
|
+
self._target_args = target_args
|
214
|
+
if '_target_kw' not in self.__dict__:
|
215
|
+
self._target_kw = target_kw
|
216
|
+
|
113
217
|
if 'label' not in self.__dict__:
|
114
218
|
if label is None:
|
115
219
|
label = (
|
@@ -188,7 +292,9 @@ class Daemon:
|
|
188
292
|
try:
|
189
293
|
os.environ['LINES'], os.environ['COLUMNS'] = str(int(lines)), str(int(columns))
|
190
294
|
with self._daemon_context:
|
295
|
+
sys.stdin = self.stdin_file
|
191
296
|
os.environ[STATIC_CONFIG['environment']['daemon_id']] = self.daemon_id
|
297
|
+
os.environ['PYTHONUNBUFFERED'] = '1'
|
192
298
|
self.rotating_log.refresh_files(start_interception=True)
|
193
299
|
result = None
|
194
300
|
try:
|
@@ -197,7 +303,7 @@ class Daemon:
|
|
197
303
|
|
198
304
|
self._log_refresh_timer.start()
|
199
305
|
self.properties['result'] = None
|
200
|
-
self.
|
306
|
+
self._capture_process_timestamp('began')
|
201
307
|
result = self.target(*self.target_args, **self.target_kw)
|
202
308
|
self.properties['result'] = result
|
203
309
|
except (BrokenPipeError, KeyboardInterrupt, SystemExit):
|
@@ -250,7 +356,7 @@ class Daemon:
|
|
250
356
|
if 'process' not in self.properties:
|
251
357
|
self.properties['process'] = {}
|
252
358
|
|
253
|
-
if process_key not in ('began', 'ended', 'paused'):
|
359
|
+
if process_key not in ('began', 'ended', 'paused', 'stopped'):
|
254
360
|
raise ValueError(f"Invalid key '{process_key}'.")
|
255
361
|
|
256
362
|
self.properties['process'][process_key] = (
|
@@ -290,6 +396,10 @@ class Daemon:
|
|
290
396
|
if self.status == 'paused':
|
291
397
|
return self.resume()
|
292
398
|
|
399
|
+
self._remove_stop_file()
|
400
|
+
if self.status == 'running':
|
401
|
+
return True, f"Daemon '{self}' is already running."
|
402
|
+
|
293
403
|
self.mkdir_if_not_exists(allow_dirty_run)
|
294
404
|
_write_pickle_success_tuple = self.write_pickle()
|
295
405
|
if not _write_pickle_success_tuple[0]:
|
@@ -312,7 +422,8 @@ class Daemon:
|
|
312
422
|
return _launch_success_bool, msg
|
313
423
|
|
314
424
|
def kill(self, timeout: Union[int, float, None] = 8) -> SuccessTuple:
|
315
|
-
"""
|
425
|
+
"""
|
426
|
+
Forcibly terminate a running daemon.
|
316
427
|
Sends a SIGTERM signal to the process.
|
317
428
|
|
318
429
|
Parameters
|
@@ -327,9 +438,11 @@ class Daemon:
|
|
327
438
|
if self.status != 'paused':
|
328
439
|
success, msg = self._send_signal(signal.SIGTERM, timeout=timeout)
|
329
440
|
if success:
|
441
|
+
self._write_stop_file('kill')
|
330
442
|
return success, msg
|
331
443
|
|
332
444
|
if self.status == 'stopped':
|
445
|
+
self._write_stop_file('kill')
|
333
446
|
return True, "Process has already stopped."
|
334
447
|
|
335
448
|
process = self.process
|
@@ -345,13 +458,18 @@ class Daemon:
|
|
345
458
|
self.pid_path.unlink()
|
346
459
|
except Exception as e:
|
347
460
|
pass
|
461
|
+
|
462
|
+
self._write_stop_file('kill')
|
348
463
|
return True, "Success"
|
349
464
|
|
350
465
|
def quit(self, timeout: Union[int, float, None] = None) -> SuccessTuple:
|
351
466
|
"""Gracefully quit a running daemon."""
|
352
467
|
if self.status == 'paused':
|
353
468
|
return self.kill(timeout)
|
469
|
+
|
354
470
|
signal_success, signal_msg = self._send_signal(signal.SIGINT, timeout=timeout)
|
471
|
+
if signal_success:
|
472
|
+
self._write_stop_file('quit')
|
355
473
|
return signal_success, signal_msg
|
356
474
|
|
357
475
|
def pause(
|
@@ -380,6 +498,7 @@ class Daemon:
|
|
380
498
|
if self.status == 'paused':
|
381
499
|
return True, f"Daemon '{self.daemon_id}' is already paused."
|
382
500
|
|
501
|
+
self._write_stop_file('pause')
|
383
502
|
try:
|
384
503
|
self.process.suspend()
|
385
504
|
except Exception as e:
|
@@ -418,10 +537,10 @@ class Daemon:
|
|
418
537
|
)
|
419
538
|
|
420
539
|
def resume(
|
421
|
-
|
422
|
-
|
423
|
-
|
424
|
-
|
540
|
+
self,
|
541
|
+
timeout: Union[int, float, None] = None,
|
542
|
+
check_timeout_interval: Union[float, int, None] = None,
|
543
|
+
) -> SuccessTuple:
|
425
544
|
"""
|
426
545
|
Resume the daemon if it is paused.
|
427
546
|
|
@@ -443,6 +562,7 @@ class Daemon:
|
|
443
562
|
if self.status == 'stopped':
|
444
563
|
return False, f"Daemon '{self.daemon_id}' is stopped and cannot be resumed."
|
445
564
|
|
565
|
+
self._remove_stop_file()
|
446
566
|
try:
|
447
567
|
self.process.resume()
|
448
568
|
except Exception as e:
|
@@ -472,6 +592,50 @@ class Daemon:
|
|
472
592
|
+ ('s' if timeout != 1 else '') + '.'
|
473
593
|
)
|
474
594
|
|
595
|
+
def _write_stop_file(self, action: str) -> SuccessTuple:
|
596
|
+
"""Write the stop file timestamp and action."""
|
597
|
+
if action not in ('quit', 'kill', 'pause'):
|
598
|
+
return False, f"Unsupported action '{action}'."
|
599
|
+
|
600
|
+
if not self.stop_path.parent.exists():
|
601
|
+
self.stop_path.parent.mkdir(parents=True, exist_ok=True)
|
602
|
+
|
603
|
+
with open(self.stop_path, 'w+', encoding='utf-8') as f:
|
604
|
+
json.dump(
|
605
|
+
{
|
606
|
+
'stop_time': datetime.now(timezone.utc).isoformat(),
|
607
|
+
'action': action,
|
608
|
+
},
|
609
|
+
f
|
610
|
+
)
|
611
|
+
|
612
|
+
return True, "Success"
|
613
|
+
|
614
|
+
def _remove_stop_file(self) -> SuccessTuple:
|
615
|
+
"""Remove the stop file"""
|
616
|
+
if not self.stop_path.exists():
|
617
|
+
return True, "Stop file does not exist."
|
618
|
+
|
619
|
+
try:
|
620
|
+
self.stop_path.unlink()
|
621
|
+
except Exception as e:
|
622
|
+
return False, f"Failed to remove stop file:\n{e}"
|
623
|
+
|
624
|
+
return True, "Success"
|
625
|
+
|
626
|
+
def _read_stop_file(self) -> Dict[str, Any]:
|
627
|
+
"""
|
628
|
+
Read the stop file if it exists.
|
629
|
+
"""
|
630
|
+
if not self.stop_path.exists():
|
631
|
+
return {}
|
632
|
+
|
633
|
+
try:
|
634
|
+
with open(self.stop_path, 'r', encoding='utf-8') as f:
|
635
|
+
data = json.load(f)
|
636
|
+
return data
|
637
|
+
except Exception:
|
638
|
+
return {}
|
475
639
|
|
476
640
|
def _handle_sigterm(self, signal_number: int, stack_frame: 'frame') -> None:
|
477
641
|
"""
|
@@ -638,6 +802,13 @@ class Daemon:
|
|
638
802
|
"""
|
639
803
|
return self._get_properties_path_from_daemon_id(self.daemon_id)
|
640
804
|
|
805
|
+
@property
|
806
|
+
def stop_path(self) -> pathlib.Path:
|
807
|
+
"""
|
808
|
+
Return the path for the stop file (created when manually stopped).
|
809
|
+
"""
|
810
|
+
return self.path / '.stop.json'
|
811
|
+
|
641
812
|
@property
|
642
813
|
def log_path(self) -> pathlib.Path:
|
643
814
|
"""
|
@@ -645,6 +816,23 @@ class Daemon:
|
|
645
816
|
"""
|
646
817
|
return LOGS_RESOURCES_PATH / (self.daemon_id + '.log')
|
647
818
|
|
819
|
+
@property
|
820
|
+
def stdin_file_path(self) -> pathlib.Path:
|
821
|
+
"""
|
822
|
+
Return the stdin file path.
|
823
|
+
"""
|
824
|
+
return self.path / 'input.stdin'
|
825
|
+
|
826
|
+
@property
|
827
|
+
def blocking_stdin_file_path(self) -> pathlib.Path:
|
828
|
+
"""
|
829
|
+
Return the stdin file path.
|
830
|
+
"""
|
831
|
+
if '_blocking_stdin_file_path' in self.__dict__:
|
832
|
+
return self._blocking_stdin_file_path
|
833
|
+
|
834
|
+
return self.path / 'input.stdin.block'
|
835
|
+
|
648
836
|
@property
|
649
837
|
def log_offset_path(self) -> pathlib.Path:
|
650
838
|
"""
|
@@ -654,20 +842,44 @@ class Daemon:
|
|
654
842
|
|
655
843
|
@property
|
656
844
|
def rotating_log(self) -> RotatingFile:
|
845
|
+
"""
|
846
|
+
The rotating log file for the daemon's output.
|
847
|
+
"""
|
657
848
|
if '_rotating_log' in self.__dict__:
|
658
849
|
return self._rotating_log
|
659
850
|
|
851
|
+
write_timestamps = (
|
852
|
+
self.properties.get('logs', {}).get('write_timestamps', None)
|
853
|
+
)
|
854
|
+
if write_timestamps is None:
|
855
|
+
write_timestamps = get_config('jobs', 'logs', 'timestamps', 'enabled')
|
856
|
+
|
660
857
|
self._rotating_log = RotatingFile(
|
661
858
|
self.log_path,
|
662
|
-
redirect_streams
|
663
|
-
write_timestamps
|
664
|
-
timestamp_format
|
859
|
+
redirect_streams=True,
|
860
|
+
write_timestamps=write_timestamps,
|
861
|
+
timestamp_format=get_config('jobs', 'logs', 'timestamps', 'format'),
|
665
862
|
)
|
666
863
|
return self._rotating_log
|
667
864
|
|
865
|
+
@property
|
866
|
+
def stdin_file(self):
|
867
|
+
"""
|
868
|
+
Return the file handler for the stdin file.
|
869
|
+
"""
|
870
|
+
if '_stdin_file' in self.__dict__:
|
871
|
+
return self._stdin_file
|
872
|
+
|
873
|
+
self._stdin_file = StdinFile(
|
874
|
+
self.stdin_file_path,
|
875
|
+
lock_file_path=self.blocking_stdin_file_path,
|
876
|
+
)
|
877
|
+
return self._stdin_file
|
878
|
+
|
668
879
|
@property
|
669
880
|
def log_text(self) -> Optional[str]:
|
670
|
-
"""
|
881
|
+
"""
|
882
|
+
Read the log files and return their contents.
|
671
883
|
Returns `None` if the log file does not exist.
|
672
884
|
"""
|
673
885
|
new_rotating_log = RotatingFile(
|
@@ -717,7 +929,8 @@ class Daemon:
|
|
717
929
|
|
718
930
|
@property
|
719
931
|
def pid(self) -> Union[int, None]:
|
720
|
-
"""
|
932
|
+
"""
|
933
|
+
Read the PID file and return its contents.
|
721
934
|
Returns `None` if the PID file does not exist.
|
722
935
|
"""
|
723
936
|
if not self.pid_path.exists():
|
@@ -725,6 +938,8 @@ class Daemon:
|
|
725
938
|
try:
|
726
939
|
with open(self.pid_path, 'r', encoding='utf-8') as f:
|
727
940
|
text = f.read()
|
941
|
+
if len(text) == 0:
|
942
|
+
return None
|
728
943
|
pid = int(text.rstrip())
|
729
944
|
except Exception as e:
|
730
945
|
warn(e)
|
@@ -764,17 +979,21 @@ class Daemon:
|
|
764
979
|
return None
|
765
980
|
try:
|
766
981
|
with open(self.properties_path, 'r', encoding='utf-8') as file:
|
767
|
-
|
982
|
+
properties = json.load(file)
|
768
983
|
except Exception as e:
|
769
|
-
|
984
|
+
properties = {}
|
985
|
+
|
986
|
+
return properties
|
770
987
|
|
771
988
|
def read_pickle(self) -> Daemon:
|
772
989
|
"""Read a Daemon's pickle file and return the `Daemon`."""
|
773
990
|
import pickle, traceback
|
774
991
|
if not self.pickle_path.exists():
|
775
992
|
error(f"Pickle file does not exist for daemon '{self.daemon_id}'.")
|
993
|
+
|
776
994
|
if self.pickle_path.stat().st_size == 0:
|
777
995
|
error(f"Pickle was empty for daemon '{self.daemon_id}'.")
|
996
|
+
|
778
997
|
try:
|
779
998
|
with open(self.pickle_path, 'rb') as pickle_file:
|
780
999
|
daemon = pickle.load(pickle_file)
|
@@ -921,9 +1140,9 @@ class Daemon:
|
|
921
1140
|
|
922
1141
|
|
923
1142
|
def get_check_timeout_interval_seconds(
|
924
|
-
|
925
|
-
|
926
|
-
|
1143
|
+
self,
|
1144
|
+
check_timeout_interval: Union[int, float, None] = None,
|
1145
|
+
) -> Union[int, float]:
|
927
1146
|
"""
|
928
1147
|
Return the interval value to check the status of timeouts.
|
929
1148
|
"""
|
@@ -931,6 +1150,33 @@ class Daemon:
|
|
931
1150
|
return check_timeout_interval
|
932
1151
|
return get_config('jobs', 'check_timeout_interval_seconds')
|
933
1152
|
|
1153
|
+
@property
|
1154
|
+
def target_args(self) -> Union[Tuple[Any], None]:
|
1155
|
+
"""
|
1156
|
+
Return the positional arguments to pass to the target function.
|
1157
|
+
"""
|
1158
|
+
target_args = (
|
1159
|
+
self.__dict__.get('_target_args', None)
|
1160
|
+
or self.properties.get('target', {}).get('args', None)
|
1161
|
+
)
|
1162
|
+
if target_args is None:
|
1163
|
+
return tuple([])
|
1164
|
+
|
1165
|
+
return tuple(target_args)
|
1166
|
+
|
1167
|
+
@property
|
1168
|
+
def target_kw(self) -> Union[Dict[str, Any], None]:
|
1169
|
+
"""
|
1170
|
+
Return the keyword arguments to pass to the target function.
|
1171
|
+
"""
|
1172
|
+
target_kw = (
|
1173
|
+
self.__dict__.get('_target_kw', None)
|
1174
|
+
or self.properties.get('target', {}).get('kw', None)
|
1175
|
+
)
|
1176
|
+
if target_kw is None:
|
1177
|
+
return {}
|
1178
|
+
|
1179
|
+
return {key: val for key, val in target_kw.items()}
|
934
1180
|
|
935
1181
|
def __getstate__(self):
|
936
1182
|
"""
|
@@ -981,4 +1227,4 @@ class Daemon:
|
|
981
1227
|
return self.daemon_id == other.daemon_id
|
982
1228
|
|
983
1229
|
def __hash__(self):
|
984
|
-
return hash(self.daemon_id)
|
1230
|
+
return hash(self.daemon_id)
|