ominfra 0.0.0.dev130__py3-none-any.whl → 0.0.0.dev132__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- ominfra/deploy/_executor.py +2 -2
- ominfra/deploy/poly/_main.py +2 -2
- ominfra/pyremote/_runcommands.py +2 -2
- ominfra/scripts/journald2aws.py +14 -6
- ominfra/scripts/supervisor.py +266 -162
- ominfra/supervisor/configs.py +17 -17
- ominfra/supervisor/dispatchersimpl.py +12 -11
- ominfra/supervisor/events.py +5 -2
- ominfra/supervisor/http.py +7 -0
- ominfra/supervisor/inject.py +8 -2
- ominfra/supervisor/io.py +2 -2
- ominfra/supervisor/main.py +13 -10
- ominfra/supervisor/pipes.py +2 -2
- ominfra/supervisor/privileges.py +4 -6
- ominfra/supervisor/processimpl.py +30 -31
- ominfra/supervisor/setupimpl.py +16 -16
- ominfra/supervisor/spawningimpl.py +1 -1
- ominfra/supervisor/states.py +11 -0
- ominfra/supervisor/supervisor.py +27 -27
- ominfra/supervisor/types.py +2 -1
- ominfra/supervisor/utils/os.py +1 -1
- ominfra/supervisor/utils/strings.py +2 -2
- {ominfra-0.0.0.dev130.dist-info → ominfra-0.0.0.dev132.dist-info}/METADATA +3 -3
- {ominfra-0.0.0.dev130.dist-info → ominfra-0.0.0.dev132.dist-info}/RECORD +28 -28
- {ominfra-0.0.0.dev130.dist-info → ominfra-0.0.0.dev132.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev130.dist-info → ominfra-0.0.0.dev132.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev130.dist-info → ominfra-0.0.0.dev132.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev130.dist-info → ominfra-0.0.0.dev132.dist-info}/top_level.txt +0 -0
ominfra/supervisor/configs.py
CHANGED
@@ -37,32 +37,32 @@ class ProcessConfig:
|
|
37
37
|
umask: ta.Optional[int] = None
|
38
38
|
priority: int = 999
|
39
39
|
|
40
|
-
|
41
|
-
|
40
|
+
auto_start: bool = True
|
41
|
+
auto_restart: str = 'unexpected'
|
42
42
|
|
43
|
-
|
44
|
-
|
43
|
+
start_secs: int = 1
|
44
|
+
start_retries: int = 3
|
45
45
|
|
46
|
-
|
47
|
-
|
46
|
+
num_procs: int = 1
|
47
|
+
num_procs_start: int = 0
|
48
48
|
|
49
49
|
@dc.dataclass(frozen=True)
|
50
50
|
class Log:
|
51
51
|
file: ta.Optional[str] = None
|
52
|
-
|
52
|
+
capture_max_bytes: ta.Optional[int] = None
|
53
53
|
events_enabled: bool = False
|
54
54
|
syslog: bool = False
|
55
55
|
backups: ta.Optional[int] = None
|
56
|
-
|
56
|
+
max_bytes: ta.Optional[int] = None
|
57
57
|
|
58
58
|
stdout: Log = Log()
|
59
59
|
stderr: Log = Log()
|
60
60
|
|
61
|
-
|
62
|
-
|
63
|
-
|
61
|
+
stop_signal: int = signal.SIGTERM
|
62
|
+
stop_wait_secs: int = 10
|
63
|
+
stop_as_group: bool = False
|
64
64
|
|
65
|
-
|
65
|
+
kill_as_group: bool = False
|
66
66
|
|
67
67
|
exitcodes: ta.Sequence[int] = (0,)
|
68
68
|
|
@@ -87,14 +87,14 @@ class ServerConfig:
|
|
87
87
|
umask: int = 0o22
|
88
88
|
directory: ta.Optional[str] = None
|
89
89
|
logfile: str = 'supervisord.log'
|
90
|
-
|
90
|
+
logfile_max_bytes: int = 50 * 1024 * 1024
|
91
91
|
logfile_backups: int = 10
|
92
92
|
loglevel: int = logging.INFO
|
93
93
|
pidfile: str = 'supervisord.pid'
|
94
94
|
identifier: str = 'supervisor'
|
95
95
|
child_logdir: str = '/dev/null'
|
96
|
-
|
97
|
-
|
96
|
+
min_fds: int = 1024
|
97
|
+
min_procs: int = 200
|
98
98
|
nocleanup: bool = False
|
99
99
|
strip_ansi: bool = False
|
100
100
|
silent: bool = False
|
@@ -107,7 +107,7 @@ class ServerConfig:
|
|
107
107
|
umask: ta.Union[int, str] = 0o22,
|
108
108
|
directory: ta.Optional[str] = None,
|
109
109
|
logfile: str = 'supervisord.log',
|
110
|
-
|
110
|
+
logfile_max_bytes: ta.Union[int, str] = 50 * 1024 * 1024,
|
111
111
|
loglevel: ta.Union[int, str] = logging.INFO,
|
112
112
|
pidfile: str = 'supervisord.pid',
|
113
113
|
child_logdir: ta.Optional[str] = None,
|
@@ -117,7 +117,7 @@ class ServerConfig:
|
|
117
117
|
umask=parse_octal(umask),
|
118
118
|
directory=check_existing_dir(directory) if directory is not None else None,
|
119
119
|
logfile=check_path_with_existing_dir(logfile),
|
120
|
-
|
120
|
+
logfile_max_bytes=parse_bytes_size(logfile_max_bytes),
|
121
121
|
loglevel=parse_logging_level(loglevel),
|
122
122
|
pidfile=check_path_with_existing_dir(pidfile),
|
123
123
|
child_logdir=child_logdir if child_logdir else tempfile.gettempdir(),
|
@@ -13,6 +13,7 @@ from .events import EventCallbacks
|
|
13
13
|
from .events import ProcessCommunicationEvent
|
14
14
|
from .events import ProcessLogStderrEvent
|
15
15
|
from .events import ProcessLogStdoutEvent
|
16
|
+
from .events import ProcessOutputChannel
|
16
17
|
from .types import Process
|
17
18
|
from .types import ProcessDispatcher
|
18
19
|
from .types import ProcessInputDispatcher
|
@@ -29,7 +30,7 @@ class BaseProcessDispatcherImpl(ProcessDispatcher, abc.ABC):
|
|
29
30
|
def __init__(
|
30
31
|
self,
|
31
32
|
process: Process,
|
32
|
-
channel:
|
33
|
+
channel: ProcessOutputChannel,
|
33
34
|
fd: Fd,
|
34
35
|
*,
|
35
36
|
event_callbacks: EventCallbacks,
|
@@ -57,7 +58,7 @@ class BaseProcessDispatcherImpl(ProcessDispatcher, abc.ABC):
|
|
57
58
|
return self._process
|
58
59
|
|
59
60
|
@property
|
60
|
-
def channel(self) ->
|
61
|
+
def channel(self) -> ProcessOutputChannel:
|
61
62
|
return self._channel
|
62
63
|
|
63
64
|
def fd(self) -> Fd:
|
@@ -147,7 +148,7 @@ class ProcessOutputDispatcherImpl(BaseProcessDispatcherImpl, ProcessOutputDispat
|
|
147
148
|
channel = self._channel # noqa
|
148
149
|
|
149
150
|
logfile = self._lc.file
|
150
|
-
|
151
|
+
max_bytes = self._lc.max_bytes # noqa
|
151
152
|
backups = self._lc.backups # noqa
|
152
153
|
to_syslog = self._lc.syslog
|
153
154
|
|
@@ -159,8 +160,8 @@ class ProcessOutputDispatcherImpl(BaseProcessDispatcherImpl, ProcessOutputDispat
|
|
159
160
|
# self.normal_log,
|
160
161
|
# filename=logfile,
|
161
162
|
# fmt='%(message)s',
|
162
|
-
# rotating=bool(
|
163
|
-
#
|
163
|
+
# rotating=bool(max_bytes), # optimization
|
164
|
+
# max_bytes=max_bytes,
|
164
165
|
# backups=backups,
|
165
166
|
# )
|
166
167
|
|
@@ -172,17 +173,17 @@ class ProcessOutputDispatcherImpl(BaseProcessDispatcherImpl, ProcessOutputDispat
|
|
172
173
|
|
173
174
|
def _init_capture_log(self) -> None:
|
174
175
|
"""
|
175
|
-
Configure the capture log for this process.
|
176
|
+
Configure the capture log for this process. This log is used to temporarily capture output when special output
|
176
177
|
is detected. Sets self.capture_log if capturing is enabled.
|
177
178
|
"""
|
178
179
|
|
179
|
-
|
180
|
-
if
|
180
|
+
capture_max_bytes = self._lc.capture_max_bytes
|
181
|
+
if capture_max_bytes:
|
181
182
|
self._capture_log = logging.getLogger(__name__)
|
182
183
|
# loggers.handle_boundIO(
|
183
184
|
# self._capture_log,
|
184
185
|
# fmt='%(message)s',
|
185
|
-
#
|
186
|
+
# max_bytes=capture_max_bytes,
|
186
187
|
# )
|
187
188
|
|
188
189
|
def remove_logs(self) -> None:
|
@@ -295,7 +296,7 @@ class ProcessOutputDispatcherImpl(BaseProcessDispatcherImpl, ProcessOutputDispat
|
|
295
296
|
self._output_buffer += data
|
296
297
|
self.record_output()
|
297
298
|
if not data:
|
298
|
-
# if we get no data back from the pipe, it means that the child process has ended.
|
299
|
+
# if we get no data back from the pipe, it means that the child process has ended. See
|
299
300
|
# mail.python.org/pipermail/python-dev/2004-August/046850.html
|
300
301
|
self.close()
|
301
302
|
|
@@ -304,7 +305,7 @@ class ProcessInputDispatcherImpl(BaseProcessDispatcherImpl, ProcessInputDispatch
|
|
304
305
|
def __init__(
|
305
306
|
self,
|
306
307
|
process: Process,
|
307
|
-
channel:
|
308
|
+
channel: ProcessOutputChannel,
|
308
309
|
fd: Fd,
|
309
310
|
*,
|
310
311
|
event_callbacks: EventCallbacks,
|
ominfra/supervisor/events.py
CHANGED
@@ -8,6 +8,9 @@ from .states import ProcessState
|
|
8
8
|
EventCallback = ta.Callable[['Event'], None]
|
9
9
|
|
10
10
|
|
11
|
+
ProcessOutputChannel = ta.Literal['stdout', 'stderr'] # ta.TypeAlias
|
12
|
+
|
13
|
+
|
11
14
|
##
|
12
15
|
|
13
16
|
|
@@ -43,7 +46,7 @@ class EventCallbacks:
|
|
43
46
|
|
44
47
|
|
45
48
|
class ProcessLogEvent(Event, abc.ABC):
|
46
|
-
channel: ta.
|
49
|
+
channel: ta.ClassVar[ProcessOutputChannel]
|
47
50
|
|
48
51
|
def __init__(self, process, pid, data):
|
49
52
|
super().__init__()
|
@@ -68,7 +71,7 @@ class ProcessCommunicationEvent(Event, abc.ABC):
|
|
68
71
|
BEGIN_TOKEN = b'<!--XSUPERVISOR:BEGIN-->'
|
69
72
|
END_TOKEN = b'<!--XSUPERVISOR:END-->'
|
70
73
|
|
71
|
-
channel: ta.ClassVar[
|
74
|
+
channel: ta.ClassVar[ProcessOutputChannel]
|
72
75
|
|
73
76
|
def __init__(self, process, pid, data):
|
74
77
|
super().__init__()
|
ominfra/supervisor/http.py
CHANGED
@@ -1,9 +1,11 @@
|
|
1
1
|
# ruff: noqa: U006 UP007
|
2
|
+
import contextlib
|
2
3
|
import json
|
3
4
|
import socket
|
4
5
|
import typing as ta
|
5
6
|
|
6
7
|
from omlish.lite.check import check_not_none
|
8
|
+
from omlish.lite.contextmanagers import defer
|
7
9
|
from omlish.lite.fdio.corohttp import CoroHttpServerConnectionFdIoHandler
|
8
10
|
from omlish.lite.fdio.handlers import SocketFdIoHandler
|
9
11
|
from omlish.lite.http.handlers import HttpHandler
|
@@ -59,6 +61,8 @@ class HttpServer(HasDispatchers):
|
|
59
61
|
self,
|
60
62
|
handler: Handler,
|
61
63
|
addr: Address = Address(('localhost', 8000)),
|
64
|
+
*,
|
65
|
+
exit_stack: contextlib.ExitStack,
|
62
66
|
) -> None:
|
63
67
|
super().__init__()
|
64
68
|
|
@@ -69,6 +73,8 @@ class HttpServer(HasDispatchers):
|
|
69
73
|
|
70
74
|
self._conns: ta.List[CoroHttpServerConnectionFdIoHandler] = []
|
71
75
|
|
76
|
+
exit_stack.enter_context(defer(self._server.close)) # noqa
|
77
|
+
|
72
78
|
def get_dispatchers(self) -> Dispatchers:
|
73
79
|
l = []
|
74
80
|
for c in self._conns:
|
@@ -113,6 +119,7 @@ class SupervisorHttpHandler:
|
|
113
119
|
'processes': {
|
114
120
|
p.name: {
|
115
121
|
'pid': p.pid,
|
122
|
+
'state': p.state.name,
|
116
123
|
}
|
117
124
|
for p in g
|
118
125
|
},
|
ominfra/supervisor/inject.py
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
# ruff: noqa: UP006 UP007
|
2
|
+
import contextlib
|
2
3
|
import dataclasses as dc
|
3
4
|
import typing as ta
|
4
5
|
|
@@ -56,6 +57,7 @@ class _FdIoPollerDaemonizeListener(DaemonizeListener):
|
|
56
57
|
|
57
58
|
|
58
59
|
def bind_server(
|
60
|
+
exit_stack: contextlib.ExitStack,
|
59
61
|
config: ServerConfig,
|
60
62
|
*,
|
61
63
|
server_epoch: ta.Optional[ServerEpoch] = None,
|
@@ -64,6 +66,8 @@ def bind_server(
|
|
64
66
|
lst: ta.List[InjectorBindingOrBindings] = [
|
65
67
|
inj.bind(config),
|
66
68
|
|
69
|
+
inj.bind(exit_stack),
|
70
|
+
|
67
71
|
inj.bind_array(DaemonizeListener),
|
68
72
|
inj.bind_array_type(DaemonizeListener, DaemonizeListeners),
|
69
73
|
|
@@ -119,8 +123,10 @@ def bind_server(
|
|
119
123
|
PollFdIoPoller,
|
120
124
|
SelectFdIoPoller,
|
121
125
|
]))
|
122
|
-
lst.
|
123
|
-
|
126
|
+
lst.extend([
|
127
|
+
inj.bind(poller_impl, key=FdIoPoller, singleton=True),
|
128
|
+
inj.bind(_FdIoPollerDaemonizeListener, array=True, singleton=True),
|
129
|
+
])
|
124
130
|
|
125
131
|
#
|
126
132
|
|
ominfra/supervisor/io.py
CHANGED
@@ -44,9 +44,9 @@ class IoManager(HasDispatchers):
|
|
44
44
|
)
|
45
45
|
|
46
46
|
timeout = 1 # this cannot be fewer than the smallest TickEvent (5)
|
47
|
-
|
47
|
+
|
48
48
|
polled = self._poller.poll(timeout)
|
49
|
-
|
49
|
+
|
50
50
|
if polled.msg is not None:
|
51
51
|
log.error(polled.msg)
|
52
52
|
if polled.exc is not None:
|
ominfra/supervisor/main.py
CHANGED
@@ -29,6 +29,7 @@
|
|
29
29
|
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
30
30
|
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
|
31
31
|
# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
32
|
+
import contextlib
|
32
33
|
import itertools
|
33
34
|
import os.path
|
34
35
|
import typing as ta
|
@@ -96,18 +97,20 @@ def main(
|
|
96
97
|
prepare=prepare_server_config,
|
97
98
|
)
|
98
99
|
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
100
|
+
with contextlib.ExitStack() as es:
|
101
|
+
injector = inj.create_injector(bind_server(
|
102
|
+
es,
|
103
|
+
config,
|
104
|
+
server_epoch=ServerEpoch(epoch),
|
105
|
+
inherited_fds=inherited_fds,
|
106
|
+
))
|
104
107
|
|
105
|
-
|
108
|
+
supervisor = injector[Supervisor]
|
106
109
|
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
110
|
+
try:
|
111
|
+
supervisor.main()
|
112
|
+
except ExitNow:
|
113
|
+
pass
|
111
114
|
|
112
115
|
if supervisor.state < SupervisorState.RESTARTING:
|
113
116
|
break
|
ominfra/supervisor/pipes.py
CHANGED
@@ -29,8 +29,8 @@ class ProcessPipes:
|
|
29
29
|
|
30
30
|
def make_process_pipes(stderr=True) -> ProcessPipes:
|
31
31
|
"""
|
32
|
-
Create pipes for parent to child stdin/stdout/stderr communications.
|
33
|
-
|
32
|
+
Create pipes for parent to child stdin/stdout/stderr communications. Open fd in non-blocking mode so we can read
|
33
|
+
them in the mainloop without blocking. If stderr is False, don't create a pipe for stderr.
|
34
34
|
"""
|
35
35
|
|
36
36
|
pipes: ta.Dict[str, ta.Optional[Fd]] = {
|
ominfra/supervisor/privileges.py
CHANGED
@@ -7,9 +7,8 @@ import typing as ta
|
|
7
7
|
|
8
8
|
def drop_privileges(user: ta.Union[int, str, None]) -> ta.Optional[str]:
|
9
9
|
"""
|
10
|
-
Drop privileges to become the specified user, which may be a username or uid.
|
11
|
-
|
12
|
-
dropped.
|
10
|
+
Drop privileges to become the specified user, which may be a username or uid. Called for supervisord startup and
|
11
|
+
when spawning subprocesses. Returns None on success or a string error message if privileges could not be dropped.
|
13
12
|
"""
|
14
13
|
|
15
14
|
if user is None:
|
@@ -33,9 +32,8 @@ def drop_privileges(user: ta.Union[int, str, None]) -> ta.Optional[str]:
|
|
33
32
|
current_uid = os.getuid()
|
34
33
|
|
35
34
|
if current_uid == uid:
|
36
|
-
# do nothing and return successfully if the uid is already the current one.
|
37
|
-
#
|
38
|
-
# it.
|
35
|
+
# do nothing and return successfully if the uid is already the current one. this allows a supervisord running as
|
36
|
+
# an unprivileged user "foo" to start a process where the config has "user=foo" (same user) in it.
|
39
37
|
return None
|
40
38
|
|
41
39
|
if current_uid != 0:
|
@@ -79,7 +79,7 @@ class ProcessImpl(Process):
|
|
79
79
|
|
80
80
|
self._killing = False # true if we are trying to kill this process
|
81
81
|
|
82
|
-
self._backoff = 0 # backoff counter (to
|
82
|
+
self._backoff = 0 # backoff counter (to start_retries)
|
83
83
|
|
84
84
|
self._exitstatus: ta.Optional[Rc] = None # status attached to dead process by finish()
|
85
85
|
self._spawn_err: ta.Optional[str] = None # error message attached by spawn() if any
|
@@ -156,7 +156,7 @@ class ProcessImpl(Process):
|
|
156
156
|
self._pipes = sp.pipes
|
157
157
|
self._dispatchers = sp.dispatchers
|
158
158
|
|
159
|
-
self._delay = time.time() + self.config.
|
159
|
+
self._delay = time.time() + self.config.start_secs
|
160
160
|
|
161
161
|
return sp.pid
|
162
162
|
|
@@ -214,17 +214,17 @@ class ProcessImpl(Process):
|
|
214
214
|
|
215
215
|
if self._state == ProcessState.STARTING:
|
216
216
|
self._last_start = min(test_time, self._last_start)
|
217
|
-
if self._delay > 0 and test_time < (self._delay - self._config.
|
218
|
-
self._delay = test_time + self._config.
|
217
|
+
if self._delay > 0 and test_time < (self._delay - self._config.start_secs):
|
218
|
+
self._delay = test_time + self._config.start_secs
|
219
219
|
|
220
220
|
elif self._state == ProcessState.RUNNING:
|
221
|
-
if test_time > self._last_start and test_time < (self._last_start + self._config.
|
222
|
-
self._last_start = test_time - self._config.
|
221
|
+
if test_time > self._last_start and test_time < (self._last_start + self._config.start_secs):
|
222
|
+
self._last_start = test_time - self._config.start_secs
|
223
223
|
|
224
224
|
elif self._state == ProcessState.STOPPING:
|
225
225
|
self._last_stop_report = min(test_time, self._last_stop_report)
|
226
|
-
if self._delay > 0 and test_time < (self._delay - self._config.
|
227
|
-
self._delay = test_time + self._config.
|
226
|
+
if self._delay > 0 and test_time < (self._delay - self._config.stop_wait_secs):
|
227
|
+
self._delay = test_time + self._config.stop_wait_secs
|
228
228
|
|
229
229
|
elif self._state == ProcessState.BACKOFF:
|
230
230
|
if self._delay > 0 and test_time < (self._delay - self._backoff):
|
@@ -233,7 +233,7 @@ class ProcessImpl(Process):
|
|
233
233
|
def stop(self) -> ta.Optional[str]:
|
234
234
|
self._administrative_stop = True
|
235
235
|
self._last_stop_report = 0
|
236
|
-
return self.kill(self._config.
|
236
|
+
return self.kill(self._config.stop_signal)
|
237
237
|
|
238
238
|
def stop_report(self) -> None:
|
239
239
|
"""Log a 'waiting for x to stop' message with throttling."""
|
@@ -256,7 +256,7 @@ class ProcessImpl(Process):
|
|
256
256
|
|
257
257
|
def kill(self, sig: int) -> ta.Optional[str]:
|
258
258
|
"""
|
259
|
-
Send a signal to the subprocess with the intention to kill it (to make it exit).
|
259
|
+
Send a signal to the subprocess with the intention to kill it (to make it exit). This may or may not actually
|
260
260
|
kill it.
|
261
261
|
|
262
262
|
Return None if the signal was sent, or an error message string if an error occurred or if the subprocess is not
|
@@ -264,8 +264,8 @@ class ProcessImpl(Process):
|
|
264
264
|
"""
|
265
265
|
now = time.time()
|
266
266
|
|
267
|
-
# If the process is in BACKOFF and we want to stop or kill it, then BACKOFF -> STOPPED.
|
268
|
-
# if
|
267
|
+
# If the process is in BACKOFF and we want to stop or kill it, then BACKOFF -> STOPPED. This is needed because
|
268
|
+
# if start_retries is a large number and the process isn't starting successfully, the stop request would be
|
269
269
|
# blocked for a long time waiting for the retries.
|
270
270
|
if self._state == ProcessState.BACKOFF:
|
271
271
|
log.debug('Attempted to kill %s, which is in BACKOFF state.', self.name)
|
@@ -280,25 +280,25 @@ class ProcessImpl(Process):
|
|
280
280
|
|
281
281
|
# If we're in the stopping state, then we've already sent the stop signal and this is the kill signal
|
282
282
|
if self._state == ProcessState.STOPPING:
|
283
|
-
|
283
|
+
kill_as_group = self._config.kill_as_group
|
284
284
|
else:
|
285
|
-
|
285
|
+
kill_as_group = self._config.stop_as_group
|
286
286
|
|
287
287
|
as_group = ''
|
288
|
-
if
|
288
|
+
if kill_as_group:
|
289
289
|
as_group = 'process group '
|
290
290
|
|
291
291
|
log.debug('killing %s (pid %s) %s with signal %s', self.name, self.pid, as_group, sig_name(sig))
|
292
292
|
|
293
293
|
# RUNNING/STARTING/STOPPING -> STOPPING
|
294
294
|
self._killing = True
|
295
|
-
self._delay = now + self._config.
|
296
|
-
# we will already be in the STOPPING state if we're doing a SIGKILL as a result of overrunning
|
295
|
+
self._delay = now + self._config.stop_wait_secs
|
296
|
+
# we will already be in the STOPPING state if we're doing a SIGKILL as a result of overrunning stop_wait_secs
|
297
297
|
self.check_in_state(ProcessState.RUNNING, ProcessState.STARTING, ProcessState.STOPPING)
|
298
298
|
self.change_state(ProcessState.STOPPING)
|
299
299
|
|
300
300
|
kpid = int(self.pid)
|
301
|
-
if
|
301
|
+
if kill_as_group:
|
302
302
|
# send to the whole process group instead
|
303
303
|
kpid = -kpid
|
304
304
|
|
@@ -308,7 +308,7 @@ class ProcessImpl(Process):
|
|
308
308
|
except OSError as exc:
|
309
309
|
if exc.errno == errno.ESRCH:
|
310
310
|
log.debug('unable to signal %s (pid %s), it probably just exited on its own: %s', self.name, self.pid, str(exc)) # noqa
|
311
|
-
# we could change the state here but we intentionally do not.
|
311
|
+
# we could change the state here but we intentionally do not. we will do it during normal SIGCHLD
|
312
312
|
# processing.
|
313
313
|
return None
|
314
314
|
raise
|
@@ -351,7 +351,7 @@ class ProcessImpl(Process):
|
|
351
351
|
self.pid,
|
352
352
|
str(exc),
|
353
353
|
)
|
354
|
-
# we could change the state here but we intentionally do not.
|
354
|
+
# we could change the state here but we intentionally do not. we will do it during normal SIGCHLD
|
355
355
|
# processing.
|
356
356
|
return None
|
357
357
|
raise
|
@@ -378,8 +378,7 @@ class ProcessImpl(Process):
|
|
378
378
|
self._last_stop = now
|
379
379
|
|
380
380
|
if now > self._last_start:
|
381
|
-
|
382
|
-
too_quickly = now - self._last_start < self._config.startsecs
|
381
|
+
too_quickly = now - self._last_start < self._config.start_secs
|
383
382
|
else:
|
384
383
|
too_quickly = False
|
385
384
|
log.warning(
|
@@ -451,8 +450,8 @@ class ProcessImpl(Process):
|
|
451
450
|
if self._supervisor_states.state > SupervisorState.RESTARTING:
|
452
451
|
# dont start any processes if supervisor is shutting down
|
453
452
|
if state == ProcessState.EXITED:
|
454
|
-
if self._config.
|
455
|
-
if self._config.
|
453
|
+
if self._config.auto_restart:
|
454
|
+
if self._config.auto_restart is RestartUnconditionally:
|
456
455
|
# EXITED -> STARTING
|
457
456
|
self.spawn()
|
458
457
|
elif self._exitstatus not in self._config.exitcodes:
|
@@ -460,29 +459,29 @@ class ProcessImpl(Process):
|
|
460
459
|
self.spawn()
|
461
460
|
|
462
461
|
elif state == ProcessState.STOPPED and not self._last_start:
|
463
|
-
if self._config.
|
462
|
+
if self._config.auto_start:
|
464
463
|
# STOPPED -> STARTING
|
465
464
|
self.spawn()
|
466
465
|
|
467
466
|
elif state == ProcessState.BACKOFF:
|
468
|
-
if self._backoff <= self._config.
|
467
|
+
if self._backoff <= self._config.start_retries:
|
469
468
|
if now > self._delay:
|
470
469
|
# BACKOFF -> STARTING
|
471
470
|
self.spawn()
|
472
471
|
|
473
472
|
if state == ProcessState.STARTING:
|
474
|
-
if now - self._last_start > self._config.
|
473
|
+
if now - self._last_start > self._config.start_secs:
|
475
474
|
# STARTING -> RUNNING if the proc has started successfully and it has stayed up for at least
|
476
|
-
# proc.config.
|
475
|
+
# proc.config.start_secs,
|
477
476
|
self._delay = 0
|
478
477
|
self._backoff = 0
|
479
478
|
self.check_in_state(ProcessState.STARTING)
|
480
479
|
self.change_state(ProcessState.RUNNING)
|
481
|
-
msg = ('entered RUNNING state, process has stayed up for > than %s seconds (
|
480
|
+
msg = ('entered RUNNING state, process has stayed up for > than %s seconds (start_secs)' % self._config.start_secs) # noqa
|
482
481
|
log.info('success: %s %s', self.name, msg)
|
483
482
|
|
484
483
|
if state == ProcessState.BACKOFF:
|
485
|
-
if self._backoff > self._config.
|
484
|
+
if self._backoff > self._config.start_retries:
|
486
485
|
# BACKOFF -> FATAL if the proc has exceeded its number of retries
|
487
486
|
self.give_up()
|
488
487
|
msg = ('entered FATAL state, too many start retries too quickly')
|
@@ -491,7 +490,7 @@ class ProcessImpl(Process):
|
|
491
490
|
elif state == ProcessState.STOPPING:
|
492
491
|
time_left = self._delay - now
|
493
492
|
if time_left <= 0:
|
494
|
-
# kill processes which are taking too long to stop with a final sigkill.
|
493
|
+
# kill processes which are taking too long to stop with a final sigkill. if this doesn't kill it, the
|
495
494
|
# process will be stuck in the STOPPING state forever.
|
496
495
|
log.warning('killing \'%s\' (%s) with SIGKILL', self.name, self.pid)
|
497
496
|
self.kill(signal.SIGKILL)
|
ominfra/supervisor/setupimpl.py
CHANGED
@@ -77,21 +77,21 @@ class SupervisorSetupImpl(SupervisorSetup):
|
|
77
77
|
def _cleanup_fds(self) -> None:
|
78
78
|
# try to close any leaked file descriptors (for reload)
|
79
79
|
start = 5
|
80
|
-
os.closerange(start, self._config.
|
80
|
+
os.closerange(start, self._config.min_fds)
|
81
81
|
|
82
82
|
#
|
83
83
|
|
84
84
|
def _set_uid_or_exit(self) -> None:
|
85
85
|
"""
|
86
|
-
Set the uid of the supervisord process.
|
86
|
+
Set the uid of the supervisord process. Called during supervisord startup only. No return value. Exits the
|
87
87
|
process via usage() if privileges could not be dropped.
|
88
88
|
"""
|
89
89
|
|
90
90
|
if self._user is None:
|
91
91
|
if os.getuid() == 0:
|
92
92
|
warnings.warn(
|
93
|
-
'Supervisor is running as root.
|
94
|
-
'config file.
|
93
|
+
'Supervisor is running as root. Privileges were not dropped because no user is specified in the '
|
94
|
+
'config file. If you intend to run as root, you can set user=root in the config file to avoid '
|
95
95
|
'this message.',
|
96
96
|
)
|
97
97
|
else:
|
@@ -105,8 +105,8 @@ class SupervisorSetupImpl(SupervisorSetup):
|
|
105
105
|
|
106
106
|
def _set_rlimits_or_exit(self) -> None:
|
107
107
|
"""
|
108
|
-
Set the rlimits of the supervisord process.
|
109
|
-
|
108
|
+
Set the rlimits of the supervisord process. Called during supervisord startup only. No return value. Exits the
|
109
|
+
process via usage() if any rlimits could not be set.
|
110
110
|
"""
|
111
111
|
|
112
112
|
limits = []
|
@@ -115,12 +115,12 @@ class SupervisorSetupImpl(SupervisorSetup):
|
|
115
115
|
limits.append({
|
116
116
|
'msg': (
|
117
117
|
'The minimum number of file descriptors required to run this process is %(min_limit)s as per the '
|
118
|
-
'"
|
119
|
-
'you to open %(hard)s file descriptors.
|
120
|
-
'your environment (see README.rst) or lower the
|
118
|
+
'"min_fds" command-line argument or config file setting. The current environment will only allow '
|
119
|
+
'you to open %(hard)s file descriptors. Either raise the number of usable file descriptors in '
|
120
|
+
'your environment (see README.rst) or lower the min_fds setting in the config file to allow the '
|
121
121
|
'process to start.'
|
122
122
|
),
|
123
|
-
'min': self._config.
|
123
|
+
'min': self._config.min_fds,
|
124
124
|
'resource': resource.RLIMIT_NOFILE,
|
125
125
|
'name': 'RLIMIT_NOFILE',
|
126
126
|
})
|
@@ -130,11 +130,11 @@ class SupervisorSetupImpl(SupervisorSetup):
|
|
130
130
|
'msg': (
|
131
131
|
'The minimum number of available processes required to run this program is %(min_limit)s as per '
|
132
132
|
'the "minprocs" command-line argument or config file setting. The current environment will only '
|
133
|
-
'allow you to open %(hard)s processes.
|
133
|
+
'allow you to open %(hard)s processes. Either raise the number of usable processes in your '
|
134
134
|
'environment (see README.rst) or lower the minprocs setting in the config file to allow the '
|
135
135
|
'program to start.'
|
136
136
|
),
|
137
|
-
'min': self._config.
|
137
|
+
'min': self._config.min_procs,
|
138
138
|
'resource': resource.RLIMIT_NPROC,
|
139
139
|
'name': 'RLIMIT_NPROC',
|
140
140
|
})
|
@@ -220,11 +220,11 @@ class SupervisorSetupImpl(SupervisorSetup):
|
|
220
220
|
dl.after_daemonize()
|
221
221
|
|
222
222
|
def _do_daemonize(self) -> None:
|
223
|
-
# To daemonize, we need to become the leader of our own session (process) group.
|
224
|
-
# our parent process will also be sent to us.
|
223
|
+
# To daemonize, we need to become the leader of our own session (process) group. If we do not, signals sent to
|
224
|
+
# our parent process will also be sent to us. This might be bad because signals such as SIGINT can be sent to
|
225
225
|
# our parent process during normal (uninteresting) operations such as when we press Ctrl-C in the parent
|
226
226
|
# terminal window to escape from a logtail command. To disassociate ourselves from our parent's session group we
|
227
|
-
# use os.setsid.
|
227
|
+
# use os.setsid. It means "set session id", which has the effect of disassociating a process from is current
|
228
228
|
# session and process group and setting itself up as a new session leader.
|
229
229
|
#
|
230
230
|
# Unfortunately we cannot call setsid if we're already a session group leader, so we use "fork" to make a copy
|
@@ -256,7 +256,7 @@ class SupervisorSetupImpl(SupervisorSetup):
|
|
256
256
|
os.dup2(2, os.open('/dev/null', os.O_WRONLY))
|
257
257
|
|
258
258
|
# XXX Stevens, in his Advanced Unix book, section 13.3 (page 417) recommends calling umask(0) and closing unused
|
259
|
-
# file descriptors.
|
259
|
+
# file descriptors. In his Network Programming book, he additionally recommends ignoring SIGHUP and forking
|
260
260
|
# again after the setsid() call, for obscure SVR4 reasons.
|
261
261
|
os.setsid()
|
262
262
|
os.umask(self._config.umask)
|
@@ -316,7 +316,7 @@ class ProcessSpawningImpl(ProcessSpawning):
|
|
316
316
|
else:
|
317
317
|
os.dup2(check_not_none(pipes.child_stderr), 2)
|
318
318
|
|
319
|
-
for i in range(3, self._server_config.
|
319
|
+
for i in range(3, self._server_config.min_fds):
|
320
320
|
if i in self._inherited_fds:
|
321
321
|
continue
|
322
322
|
close_fd(Fd(i))
|
ominfra/supervisor/states.py
CHANGED
@@ -27,6 +27,17 @@ class ProcessState(enum.IntEnum):
|
|
27
27
|
return self in SIGNALABLE_STATES
|
28
28
|
|
29
29
|
|
30
|
+
# http://supervisord.org/subprocess.html
|
31
|
+
STATE_TRANSITIONS = {
|
32
|
+
ProcessState.STOPPED: (ProcessState.STARTING,),
|
33
|
+
ProcessState.STARTING: (ProcessState.RUNNING, ProcessState.BACKOFF, ProcessState.STOPPING),
|
34
|
+
ProcessState.RUNNING: (ProcessState.STOPPING, ProcessState.EXITED),
|
35
|
+
ProcessState.BACKOFF: (ProcessState.STARTING, ProcessState.FATAL),
|
36
|
+
ProcessState.STOPPING: (ProcessState.STOPPED,),
|
37
|
+
ProcessState.EXITED: (ProcessState.STARTING,),
|
38
|
+
ProcessState.FATAL: (ProcessState.STARTING,),
|
39
|
+
}
|
40
|
+
|
30
41
|
STOPPED_STATES = (
|
31
42
|
ProcessState.STOPPED,
|
32
43
|
ProcessState.EXITED,
|