ominfra 0.0.0.dev130__py3-none-any.whl → 0.0.0.dev132__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ominfra/deploy/_executor.py +2 -2
- ominfra/deploy/poly/_main.py +2 -2
- ominfra/pyremote/_runcommands.py +2 -2
- ominfra/scripts/journald2aws.py +14 -6
- ominfra/scripts/supervisor.py +266 -162
- ominfra/supervisor/configs.py +17 -17
- ominfra/supervisor/dispatchersimpl.py +12 -11
- ominfra/supervisor/events.py +5 -2
- ominfra/supervisor/http.py +7 -0
- ominfra/supervisor/inject.py +8 -2
- ominfra/supervisor/io.py +2 -2
- ominfra/supervisor/main.py +13 -10
- ominfra/supervisor/pipes.py +2 -2
- ominfra/supervisor/privileges.py +4 -6
- ominfra/supervisor/processimpl.py +30 -31
- ominfra/supervisor/setupimpl.py +16 -16
- ominfra/supervisor/spawningimpl.py +1 -1
- ominfra/supervisor/states.py +11 -0
- ominfra/supervisor/supervisor.py +27 -27
- ominfra/supervisor/types.py +2 -1
- ominfra/supervisor/utils/os.py +1 -1
- ominfra/supervisor/utils/strings.py +2 -2
- {ominfra-0.0.0.dev130.dist-info → ominfra-0.0.0.dev132.dist-info}/METADATA +3 -3
- {ominfra-0.0.0.dev130.dist-info → ominfra-0.0.0.dev132.dist-info}/RECORD +28 -28
- {ominfra-0.0.0.dev130.dist-info → ominfra-0.0.0.dev132.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev130.dist-info → ominfra-0.0.0.dev132.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev130.dist-info → ominfra-0.0.0.dev132.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev130.dist-info → ominfra-0.0.0.dev132.dist-info}/top_level.txt +0 -0
ominfra/supervisor/configs.py
CHANGED
@@ -37,32 +37,32 @@ class ProcessConfig:
|
|
37
37
|
umask: ta.Optional[int] = None
|
38
38
|
priority: int = 999
|
39
39
|
|
40
|
-
|
41
|
-
|
40
|
+
auto_start: bool = True
|
41
|
+
auto_restart: str = 'unexpected'
|
42
42
|
|
43
|
-
|
44
|
-
|
43
|
+
start_secs: int = 1
|
44
|
+
start_retries: int = 3
|
45
45
|
|
46
|
-
|
47
|
-
|
46
|
+
num_procs: int = 1
|
47
|
+
num_procs_start: int = 0
|
48
48
|
|
49
49
|
@dc.dataclass(frozen=True)
|
50
50
|
class Log:
|
51
51
|
file: ta.Optional[str] = None
|
52
|
-
|
52
|
+
capture_max_bytes: ta.Optional[int] = None
|
53
53
|
events_enabled: bool = False
|
54
54
|
syslog: bool = False
|
55
55
|
backups: ta.Optional[int] = None
|
56
|
-
|
56
|
+
max_bytes: ta.Optional[int] = None
|
57
57
|
|
58
58
|
stdout: Log = Log()
|
59
59
|
stderr: Log = Log()
|
60
60
|
|
61
|
-
|
62
|
-
|
63
|
-
|
61
|
+
stop_signal: int = signal.SIGTERM
|
62
|
+
stop_wait_secs: int = 10
|
63
|
+
stop_as_group: bool = False
|
64
64
|
|
65
|
-
|
65
|
+
kill_as_group: bool = False
|
66
66
|
|
67
67
|
exitcodes: ta.Sequence[int] = (0,)
|
68
68
|
|
@@ -87,14 +87,14 @@ class ServerConfig:
|
|
87
87
|
umask: int = 0o22
|
88
88
|
directory: ta.Optional[str] = None
|
89
89
|
logfile: str = 'supervisord.log'
|
90
|
-
|
90
|
+
logfile_max_bytes: int = 50 * 1024 * 1024
|
91
91
|
logfile_backups: int = 10
|
92
92
|
loglevel: int = logging.INFO
|
93
93
|
pidfile: str = 'supervisord.pid'
|
94
94
|
identifier: str = 'supervisor'
|
95
95
|
child_logdir: str = '/dev/null'
|
96
|
-
|
97
|
-
|
96
|
+
min_fds: int = 1024
|
97
|
+
min_procs: int = 200
|
98
98
|
nocleanup: bool = False
|
99
99
|
strip_ansi: bool = False
|
100
100
|
silent: bool = False
|
@@ -107,7 +107,7 @@ class ServerConfig:
|
|
107
107
|
umask: ta.Union[int, str] = 0o22,
|
108
108
|
directory: ta.Optional[str] = None,
|
109
109
|
logfile: str = 'supervisord.log',
|
110
|
-
|
110
|
+
logfile_max_bytes: ta.Union[int, str] = 50 * 1024 * 1024,
|
111
111
|
loglevel: ta.Union[int, str] = logging.INFO,
|
112
112
|
pidfile: str = 'supervisord.pid',
|
113
113
|
child_logdir: ta.Optional[str] = None,
|
@@ -117,7 +117,7 @@ class ServerConfig:
|
|
117
117
|
umask=parse_octal(umask),
|
118
118
|
directory=check_existing_dir(directory) if directory is not None else None,
|
119
119
|
logfile=check_path_with_existing_dir(logfile),
|
120
|
-
|
120
|
+
logfile_max_bytes=parse_bytes_size(logfile_max_bytes),
|
121
121
|
loglevel=parse_logging_level(loglevel),
|
122
122
|
pidfile=check_path_with_existing_dir(pidfile),
|
123
123
|
child_logdir=child_logdir if child_logdir else tempfile.gettempdir(),
|
@@ -13,6 +13,7 @@ from .events import EventCallbacks
|
|
13
13
|
from .events import ProcessCommunicationEvent
|
14
14
|
from .events import ProcessLogStderrEvent
|
15
15
|
from .events import ProcessLogStdoutEvent
|
16
|
+
from .events import ProcessOutputChannel
|
16
17
|
from .types import Process
|
17
18
|
from .types import ProcessDispatcher
|
18
19
|
from .types import ProcessInputDispatcher
|
@@ -29,7 +30,7 @@ class BaseProcessDispatcherImpl(ProcessDispatcher, abc.ABC):
|
|
29
30
|
def __init__(
|
30
31
|
self,
|
31
32
|
process: Process,
|
32
|
-
channel:
|
33
|
+
channel: ProcessOutputChannel,
|
33
34
|
fd: Fd,
|
34
35
|
*,
|
35
36
|
event_callbacks: EventCallbacks,
|
@@ -57,7 +58,7 @@ class BaseProcessDispatcherImpl(ProcessDispatcher, abc.ABC):
|
|
57
58
|
return self._process
|
58
59
|
|
59
60
|
@property
|
60
|
-
def channel(self) ->
|
61
|
+
def channel(self) -> ProcessOutputChannel:
|
61
62
|
return self._channel
|
62
63
|
|
63
64
|
def fd(self) -> Fd:
|
@@ -147,7 +148,7 @@ class ProcessOutputDispatcherImpl(BaseProcessDispatcherImpl, ProcessOutputDispat
|
|
147
148
|
channel = self._channel # noqa
|
148
149
|
|
149
150
|
logfile = self._lc.file
|
150
|
-
|
151
|
+
max_bytes = self._lc.max_bytes # noqa
|
151
152
|
backups = self._lc.backups # noqa
|
152
153
|
to_syslog = self._lc.syslog
|
153
154
|
|
@@ -159,8 +160,8 @@ class ProcessOutputDispatcherImpl(BaseProcessDispatcherImpl, ProcessOutputDispat
|
|
159
160
|
# self.normal_log,
|
160
161
|
# filename=logfile,
|
161
162
|
# fmt='%(message)s',
|
162
|
-
# rotating=bool(
|
163
|
-
#
|
163
|
+
# rotating=bool(max_bytes), # optimization
|
164
|
+
# max_bytes=max_bytes,
|
164
165
|
# backups=backups,
|
165
166
|
# )
|
166
167
|
|
@@ -172,17 +173,17 @@ class ProcessOutputDispatcherImpl(BaseProcessDispatcherImpl, ProcessOutputDispat
|
|
172
173
|
|
173
174
|
def _init_capture_log(self) -> None:
|
174
175
|
"""
|
175
|
-
Configure the capture log for this process.
|
176
|
+
Configure the capture log for this process. This log is used to temporarily capture output when special output
|
176
177
|
is detected. Sets self.capture_log if capturing is enabled.
|
177
178
|
"""
|
178
179
|
|
179
|
-
|
180
|
-
if
|
180
|
+
capture_max_bytes = self._lc.capture_max_bytes
|
181
|
+
if capture_max_bytes:
|
181
182
|
self._capture_log = logging.getLogger(__name__)
|
182
183
|
# loggers.handle_boundIO(
|
183
184
|
# self._capture_log,
|
184
185
|
# fmt='%(message)s',
|
185
|
-
#
|
186
|
+
# max_bytes=capture_max_bytes,
|
186
187
|
# )
|
187
188
|
|
188
189
|
def remove_logs(self) -> None:
|
@@ -295,7 +296,7 @@ class ProcessOutputDispatcherImpl(BaseProcessDispatcherImpl, ProcessOutputDispat
|
|
295
296
|
self._output_buffer += data
|
296
297
|
self.record_output()
|
297
298
|
if not data:
|
298
|
-
# if we get no data back from the pipe, it means that the child process has ended.
|
299
|
+
# if we get no data back from the pipe, it means that the child process has ended. See
|
299
300
|
# mail.python.org/pipermail/python-dev/2004-August/046850.html
|
300
301
|
self.close()
|
301
302
|
|
@@ -304,7 +305,7 @@ class ProcessInputDispatcherImpl(BaseProcessDispatcherImpl, ProcessInputDispatch
|
|
304
305
|
def __init__(
|
305
306
|
self,
|
306
307
|
process: Process,
|
307
|
-
channel:
|
308
|
+
channel: ProcessOutputChannel,
|
308
309
|
fd: Fd,
|
309
310
|
*,
|
310
311
|
event_callbacks: EventCallbacks,
|
ominfra/supervisor/events.py
CHANGED
@@ -8,6 +8,9 @@ from .states import ProcessState
|
|
8
8
|
EventCallback = ta.Callable[['Event'], None]
|
9
9
|
|
10
10
|
|
11
|
+
ProcessOutputChannel = ta.Literal['stdout', 'stderr'] # ta.TypeAlias
|
12
|
+
|
13
|
+
|
11
14
|
##
|
12
15
|
|
13
16
|
|
@@ -43,7 +46,7 @@ class EventCallbacks:
|
|
43
46
|
|
44
47
|
|
45
48
|
class ProcessLogEvent(Event, abc.ABC):
|
46
|
-
channel: ta.
|
49
|
+
channel: ta.ClassVar[ProcessOutputChannel]
|
47
50
|
|
48
51
|
def __init__(self, process, pid, data):
|
49
52
|
super().__init__()
|
@@ -68,7 +71,7 @@ class ProcessCommunicationEvent(Event, abc.ABC):
|
|
68
71
|
BEGIN_TOKEN = b'<!--XSUPERVISOR:BEGIN-->'
|
69
72
|
END_TOKEN = b'<!--XSUPERVISOR:END-->'
|
70
73
|
|
71
|
-
channel: ta.ClassVar[
|
74
|
+
channel: ta.ClassVar[ProcessOutputChannel]
|
72
75
|
|
73
76
|
def __init__(self, process, pid, data):
|
74
77
|
super().__init__()
|
ominfra/supervisor/http.py
CHANGED
@@ -1,9 +1,11 @@
|
|
1
1
|
# ruff: noqa: U006 UP007
|
2
|
+
import contextlib
|
2
3
|
import json
|
3
4
|
import socket
|
4
5
|
import typing as ta
|
5
6
|
|
6
7
|
from omlish.lite.check import check_not_none
|
8
|
+
from omlish.lite.contextmanagers import defer
|
7
9
|
from omlish.lite.fdio.corohttp import CoroHttpServerConnectionFdIoHandler
|
8
10
|
from omlish.lite.fdio.handlers import SocketFdIoHandler
|
9
11
|
from omlish.lite.http.handlers import HttpHandler
|
@@ -59,6 +61,8 @@ class HttpServer(HasDispatchers):
|
|
59
61
|
self,
|
60
62
|
handler: Handler,
|
61
63
|
addr: Address = Address(('localhost', 8000)),
|
64
|
+
*,
|
65
|
+
exit_stack: contextlib.ExitStack,
|
62
66
|
) -> None:
|
63
67
|
super().__init__()
|
64
68
|
|
@@ -69,6 +73,8 @@ class HttpServer(HasDispatchers):
|
|
69
73
|
|
70
74
|
self._conns: ta.List[CoroHttpServerConnectionFdIoHandler] = []
|
71
75
|
|
76
|
+
exit_stack.enter_context(defer(self._server.close)) # noqa
|
77
|
+
|
72
78
|
def get_dispatchers(self) -> Dispatchers:
|
73
79
|
l = []
|
74
80
|
for c in self._conns:
|
@@ -113,6 +119,7 @@ class SupervisorHttpHandler:
|
|
113
119
|
'processes': {
|
114
120
|
p.name: {
|
115
121
|
'pid': p.pid,
|
122
|
+
'state': p.state.name,
|
116
123
|
}
|
117
124
|
for p in g
|
118
125
|
},
|
ominfra/supervisor/inject.py
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
# ruff: noqa: UP006 UP007
|
2
|
+
import contextlib
|
2
3
|
import dataclasses as dc
|
3
4
|
import typing as ta
|
4
5
|
|
@@ -56,6 +57,7 @@ class _FdIoPollerDaemonizeListener(DaemonizeListener):
|
|
56
57
|
|
57
58
|
|
58
59
|
def bind_server(
|
60
|
+
exit_stack: contextlib.ExitStack,
|
59
61
|
config: ServerConfig,
|
60
62
|
*,
|
61
63
|
server_epoch: ta.Optional[ServerEpoch] = None,
|
@@ -64,6 +66,8 @@ def bind_server(
|
|
64
66
|
lst: ta.List[InjectorBindingOrBindings] = [
|
65
67
|
inj.bind(config),
|
66
68
|
|
69
|
+
inj.bind(exit_stack),
|
70
|
+
|
67
71
|
inj.bind_array(DaemonizeListener),
|
68
72
|
inj.bind_array_type(DaemonizeListener, DaemonizeListeners),
|
69
73
|
|
@@ -119,8 +123,10 @@ def bind_server(
|
|
119
123
|
PollFdIoPoller,
|
120
124
|
SelectFdIoPoller,
|
121
125
|
]))
|
122
|
-
lst.
|
123
|
-
|
126
|
+
lst.extend([
|
127
|
+
inj.bind(poller_impl, key=FdIoPoller, singleton=True),
|
128
|
+
inj.bind(_FdIoPollerDaemonizeListener, array=True, singleton=True),
|
129
|
+
])
|
124
130
|
|
125
131
|
#
|
126
132
|
|
ominfra/supervisor/io.py
CHANGED
@@ -44,9 +44,9 @@ class IoManager(HasDispatchers):
|
|
44
44
|
)
|
45
45
|
|
46
46
|
timeout = 1 # this cannot be fewer than the smallest TickEvent (5)
|
47
|
-
|
47
|
+
|
48
48
|
polled = self._poller.poll(timeout)
|
49
|
-
|
49
|
+
|
50
50
|
if polled.msg is not None:
|
51
51
|
log.error(polled.msg)
|
52
52
|
if polled.exc is not None:
|
ominfra/supervisor/main.py
CHANGED
@@ -29,6 +29,7 @@
|
|
29
29
|
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
30
30
|
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
|
31
31
|
# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
32
|
+
import contextlib
|
32
33
|
import itertools
|
33
34
|
import os.path
|
34
35
|
import typing as ta
|
@@ -96,18 +97,20 @@ def main(
|
|
96
97
|
prepare=prepare_server_config,
|
97
98
|
)
|
98
99
|
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
100
|
+
with contextlib.ExitStack() as es:
|
101
|
+
injector = inj.create_injector(bind_server(
|
102
|
+
es,
|
103
|
+
config,
|
104
|
+
server_epoch=ServerEpoch(epoch),
|
105
|
+
inherited_fds=inherited_fds,
|
106
|
+
))
|
104
107
|
|
105
|
-
|
108
|
+
supervisor = injector[Supervisor]
|
106
109
|
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
110
|
+
try:
|
111
|
+
supervisor.main()
|
112
|
+
except ExitNow:
|
113
|
+
pass
|
111
114
|
|
112
115
|
if supervisor.state < SupervisorState.RESTARTING:
|
113
116
|
break
|
ominfra/supervisor/pipes.py
CHANGED
@@ -29,8 +29,8 @@ class ProcessPipes:
|
|
29
29
|
|
30
30
|
def make_process_pipes(stderr=True) -> ProcessPipes:
|
31
31
|
"""
|
32
|
-
Create pipes for parent to child stdin/stdout/stderr communications.
|
33
|
-
|
32
|
+
Create pipes for parent to child stdin/stdout/stderr communications. Open fd in non-blocking mode so we can read
|
33
|
+
them in the mainloop without blocking. If stderr is False, don't create a pipe for stderr.
|
34
34
|
"""
|
35
35
|
|
36
36
|
pipes: ta.Dict[str, ta.Optional[Fd]] = {
|
ominfra/supervisor/privileges.py
CHANGED
@@ -7,9 +7,8 @@ import typing as ta
|
|
7
7
|
|
8
8
|
def drop_privileges(user: ta.Union[int, str, None]) -> ta.Optional[str]:
|
9
9
|
"""
|
10
|
-
Drop privileges to become the specified user, which may be a username or uid.
|
11
|
-
|
12
|
-
dropped.
|
10
|
+
Drop privileges to become the specified user, which may be a username or uid. Called for supervisord startup and
|
11
|
+
when spawning subprocesses. Returns None on success or a string error message if privileges could not be dropped.
|
13
12
|
"""
|
14
13
|
|
15
14
|
if user is None:
|
@@ -33,9 +32,8 @@ def drop_privileges(user: ta.Union[int, str, None]) -> ta.Optional[str]:
|
|
33
32
|
current_uid = os.getuid()
|
34
33
|
|
35
34
|
if current_uid == uid:
|
36
|
-
# do nothing and return successfully if the uid is already the current one.
|
37
|
-
#
|
38
|
-
# it.
|
35
|
+
# do nothing and return successfully if the uid is already the current one. this allows a supervisord running as
|
36
|
+
# an unprivileged user "foo" to start a process where the config has "user=foo" (same user) in it.
|
39
37
|
return None
|
40
38
|
|
41
39
|
if current_uid != 0:
|
@@ -79,7 +79,7 @@ class ProcessImpl(Process):
|
|
79
79
|
|
80
80
|
self._killing = False # true if we are trying to kill this process
|
81
81
|
|
82
|
-
self._backoff = 0 # backoff counter (to
|
82
|
+
self._backoff = 0 # backoff counter (to start_retries)
|
83
83
|
|
84
84
|
self._exitstatus: ta.Optional[Rc] = None # status attached to dead process by finish()
|
85
85
|
self._spawn_err: ta.Optional[str] = None # error message attached by spawn() if any
|
@@ -156,7 +156,7 @@ class ProcessImpl(Process):
|
|
156
156
|
self._pipes = sp.pipes
|
157
157
|
self._dispatchers = sp.dispatchers
|
158
158
|
|
159
|
-
self._delay = time.time() + self.config.
|
159
|
+
self._delay = time.time() + self.config.start_secs
|
160
160
|
|
161
161
|
return sp.pid
|
162
162
|
|
@@ -214,17 +214,17 @@ class ProcessImpl(Process):
|
|
214
214
|
|
215
215
|
if self._state == ProcessState.STARTING:
|
216
216
|
self._last_start = min(test_time, self._last_start)
|
217
|
-
if self._delay > 0 and test_time < (self._delay - self._config.
|
218
|
-
self._delay = test_time + self._config.
|
217
|
+
if self._delay > 0 and test_time < (self._delay - self._config.start_secs):
|
218
|
+
self._delay = test_time + self._config.start_secs
|
219
219
|
|
220
220
|
elif self._state == ProcessState.RUNNING:
|
221
|
-
if test_time > self._last_start and test_time < (self._last_start + self._config.
|
222
|
-
self._last_start = test_time - self._config.
|
221
|
+
if test_time > self._last_start and test_time < (self._last_start + self._config.start_secs):
|
222
|
+
self._last_start = test_time - self._config.start_secs
|
223
223
|
|
224
224
|
elif self._state == ProcessState.STOPPING:
|
225
225
|
self._last_stop_report = min(test_time, self._last_stop_report)
|
226
|
-
if self._delay > 0 and test_time < (self._delay - self._config.
|
227
|
-
self._delay = test_time + self._config.
|
226
|
+
if self._delay > 0 and test_time < (self._delay - self._config.stop_wait_secs):
|
227
|
+
self._delay = test_time + self._config.stop_wait_secs
|
228
228
|
|
229
229
|
elif self._state == ProcessState.BACKOFF:
|
230
230
|
if self._delay > 0 and test_time < (self._delay - self._backoff):
|
@@ -233,7 +233,7 @@ class ProcessImpl(Process):
|
|
233
233
|
def stop(self) -> ta.Optional[str]:
|
234
234
|
self._administrative_stop = True
|
235
235
|
self._last_stop_report = 0
|
236
|
-
return self.kill(self._config.
|
236
|
+
return self.kill(self._config.stop_signal)
|
237
237
|
|
238
238
|
def stop_report(self) -> None:
|
239
239
|
"""Log a 'waiting for x to stop' message with throttling."""
|
@@ -256,7 +256,7 @@ class ProcessImpl(Process):
|
|
256
256
|
|
257
257
|
def kill(self, sig: int) -> ta.Optional[str]:
|
258
258
|
"""
|
259
|
-
Send a signal to the subprocess with the intention to kill it (to make it exit).
|
259
|
+
Send a signal to the subprocess with the intention to kill it (to make it exit). This may or may not actually
|
260
260
|
kill it.
|
261
261
|
|
262
262
|
Return None if the signal was sent, or an error message string if an error occurred or if the subprocess is not
|
@@ -264,8 +264,8 @@ class ProcessImpl(Process):
|
|
264
264
|
"""
|
265
265
|
now = time.time()
|
266
266
|
|
267
|
-
# If the process is in BACKOFF and we want to stop or kill it, then BACKOFF -> STOPPED.
|
268
|
-
# if
|
267
|
+
# If the process is in BACKOFF and we want to stop or kill it, then BACKOFF -> STOPPED. This is needed because
|
268
|
+
# if start_retries is a large number and the process isn't starting successfully, the stop request would be
|
269
269
|
# blocked for a long time waiting for the retries.
|
270
270
|
if self._state == ProcessState.BACKOFF:
|
271
271
|
log.debug('Attempted to kill %s, which is in BACKOFF state.', self.name)
|
@@ -280,25 +280,25 @@ class ProcessImpl(Process):
|
|
280
280
|
|
281
281
|
# If we're in the stopping state, then we've already sent the stop signal and this is the kill signal
|
282
282
|
if self._state == ProcessState.STOPPING:
|
283
|
-
|
283
|
+
kill_as_group = self._config.kill_as_group
|
284
284
|
else:
|
285
|
-
|
285
|
+
kill_as_group = self._config.stop_as_group
|
286
286
|
|
287
287
|
as_group = ''
|
288
|
-
if
|
288
|
+
if kill_as_group:
|
289
289
|
as_group = 'process group '
|
290
290
|
|
291
291
|
log.debug('killing %s (pid %s) %s with signal %s', self.name, self.pid, as_group, sig_name(sig))
|
292
292
|
|
293
293
|
# RUNNING/STARTING/STOPPING -> STOPPING
|
294
294
|
self._killing = True
|
295
|
-
self._delay = now + self._config.
|
296
|
-
# we will already be in the STOPPING state if we're doing a SIGKILL as a result of overrunning
|
295
|
+
self._delay = now + self._config.stop_wait_secs
|
296
|
+
# we will already be in the STOPPING state if we're doing a SIGKILL as a result of overrunning stop_wait_secs
|
297
297
|
self.check_in_state(ProcessState.RUNNING, ProcessState.STARTING, ProcessState.STOPPING)
|
298
298
|
self.change_state(ProcessState.STOPPING)
|
299
299
|
|
300
300
|
kpid = int(self.pid)
|
301
|
-
if
|
301
|
+
if kill_as_group:
|
302
302
|
# send to the whole process group instead
|
303
303
|
kpid = -kpid
|
304
304
|
|
@@ -308,7 +308,7 @@ class ProcessImpl(Process):
|
|
308
308
|
except OSError as exc:
|
309
309
|
if exc.errno == errno.ESRCH:
|
310
310
|
log.debug('unable to signal %s (pid %s), it probably just exited on its own: %s', self.name, self.pid, str(exc)) # noqa
|
311
|
-
# we could change the state here but we intentionally do not.
|
311
|
+
# we could change the state here but we intentionally do not. we will do it during normal SIGCHLD
|
312
312
|
# processing.
|
313
313
|
return None
|
314
314
|
raise
|
@@ -351,7 +351,7 @@ class ProcessImpl(Process):
|
|
351
351
|
self.pid,
|
352
352
|
str(exc),
|
353
353
|
)
|
354
|
-
# we could change the state here but we intentionally do not.
|
354
|
+
# we could change the state here but we intentionally do not. we will do it during normal SIGCHLD
|
355
355
|
# processing.
|
356
356
|
return None
|
357
357
|
raise
|
@@ -378,8 +378,7 @@ class ProcessImpl(Process):
|
|
378
378
|
self._last_stop = now
|
379
379
|
|
380
380
|
if now > self._last_start:
|
381
|
-
|
382
|
-
too_quickly = now - self._last_start < self._config.startsecs
|
381
|
+
too_quickly = now - self._last_start < self._config.start_secs
|
383
382
|
else:
|
384
383
|
too_quickly = False
|
385
384
|
log.warning(
|
@@ -451,8 +450,8 @@ class ProcessImpl(Process):
|
|
451
450
|
if self._supervisor_states.state > SupervisorState.RESTARTING:
|
452
451
|
# dont start any processes if supervisor is shutting down
|
453
452
|
if state == ProcessState.EXITED:
|
454
|
-
if self._config.
|
455
|
-
if self._config.
|
453
|
+
if self._config.auto_restart:
|
454
|
+
if self._config.auto_restart is RestartUnconditionally:
|
456
455
|
# EXITED -> STARTING
|
457
456
|
self.spawn()
|
458
457
|
elif self._exitstatus not in self._config.exitcodes:
|
@@ -460,29 +459,29 @@ class ProcessImpl(Process):
|
|
460
459
|
self.spawn()
|
461
460
|
|
462
461
|
elif state == ProcessState.STOPPED and not self._last_start:
|
463
|
-
if self._config.
|
462
|
+
if self._config.auto_start:
|
464
463
|
# STOPPED -> STARTING
|
465
464
|
self.spawn()
|
466
465
|
|
467
466
|
elif state == ProcessState.BACKOFF:
|
468
|
-
if self._backoff <= self._config.
|
467
|
+
if self._backoff <= self._config.start_retries:
|
469
468
|
if now > self._delay:
|
470
469
|
# BACKOFF -> STARTING
|
471
470
|
self.spawn()
|
472
471
|
|
473
472
|
if state == ProcessState.STARTING:
|
474
|
-
if now - self._last_start > self._config.
|
473
|
+
if now - self._last_start > self._config.start_secs:
|
475
474
|
# STARTING -> RUNNING if the proc has started successfully and it has stayed up for at least
|
476
|
-
# proc.config.
|
475
|
+
# proc.config.start_secs,
|
477
476
|
self._delay = 0
|
478
477
|
self._backoff = 0
|
479
478
|
self.check_in_state(ProcessState.STARTING)
|
480
479
|
self.change_state(ProcessState.RUNNING)
|
481
|
-
msg = ('entered RUNNING state, process has stayed up for > than %s seconds (
|
480
|
+
msg = ('entered RUNNING state, process has stayed up for > than %s seconds (start_secs)' % self._config.start_secs) # noqa
|
482
481
|
log.info('success: %s %s', self.name, msg)
|
483
482
|
|
484
483
|
if state == ProcessState.BACKOFF:
|
485
|
-
if self._backoff > self._config.
|
484
|
+
if self._backoff > self._config.start_retries:
|
486
485
|
# BACKOFF -> FATAL if the proc has exceeded its number of retries
|
487
486
|
self.give_up()
|
488
487
|
msg = ('entered FATAL state, too many start retries too quickly')
|
@@ -491,7 +490,7 @@ class ProcessImpl(Process):
|
|
491
490
|
elif state == ProcessState.STOPPING:
|
492
491
|
time_left = self._delay - now
|
493
492
|
if time_left <= 0:
|
494
|
-
# kill processes which are taking too long to stop with a final sigkill.
|
493
|
+
# kill processes which are taking too long to stop with a final sigkill. if this doesn't kill it, the
|
495
494
|
# process will be stuck in the STOPPING state forever.
|
496
495
|
log.warning('killing \'%s\' (%s) with SIGKILL', self.name, self.pid)
|
497
496
|
self.kill(signal.SIGKILL)
|
ominfra/supervisor/setupimpl.py
CHANGED
@@ -77,21 +77,21 @@ class SupervisorSetupImpl(SupervisorSetup):
|
|
77
77
|
def _cleanup_fds(self) -> None:
|
78
78
|
# try to close any leaked file descriptors (for reload)
|
79
79
|
start = 5
|
80
|
-
os.closerange(start, self._config.
|
80
|
+
os.closerange(start, self._config.min_fds)
|
81
81
|
|
82
82
|
#
|
83
83
|
|
84
84
|
def _set_uid_or_exit(self) -> None:
|
85
85
|
"""
|
86
|
-
Set the uid of the supervisord process.
|
86
|
+
Set the uid of the supervisord process. Called during supervisord startup only. No return value. Exits the
|
87
87
|
process via usage() if privileges could not be dropped.
|
88
88
|
"""
|
89
89
|
|
90
90
|
if self._user is None:
|
91
91
|
if os.getuid() == 0:
|
92
92
|
warnings.warn(
|
93
|
-
'Supervisor is running as root.
|
94
|
-
'config file.
|
93
|
+
'Supervisor is running as root. Privileges were not dropped because no user is specified in the '
|
94
|
+
'config file. If you intend to run as root, you can set user=root in the config file to avoid '
|
95
95
|
'this message.',
|
96
96
|
)
|
97
97
|
else:
|
@@ -105,8 +105,8 @@ class SupervisorSetupImpl(SupervisorSetup):
|
|
105
105
|
|
106
106
|
def _set_rlimits_or_exit(self) -> None:
|
107
107
|
"""
|
108
|
-
Set the rlimits of the supervisord process.
|
109
|
-
|
108
|
+
Set the rlimits of the supervisord process. Called during supervisord startup only. No return value. Exits the
|
109
|
+
process via usage() if any rlimits could not be set.
|
110
110
|
"""
|
111
111
|
|
112
112
|
limits = []
|
@@ -115,12 +115,12 @@ class SupervisorSetupImpl(SupervisorSetup):
|
|
115
115
|
limits.append({
|
116
116
|
'msg': (
|
117
117
|
'The minimum number of file descriptors required to run this process is %(min_limit)s as per the '
|
118
|
-
'"
|
119
|
-
'you to open %(hard)s file descriptors.
|
120
|
-
'your environment (see README.rst) or lower the
|
118
|
+
'"min_fds" command-line argument or config file setting. The current environment will only allow '
|
119
|
+
'you to open %(hard)s file descriptors. Either raise the number of usable file descriptors in '
|
120
|
+
'your environment (see README.rst) or lower the min_fds setting in the config file to allow the '
|
121
121
|
'process to start.'
|
122
122
|
),
|
123
|
-
'min': self._config.
|
123
|
+
'min': self._config.min_fds,
|
124
124
|
'resource': resource.RLIMIT_NOFILE,
|
125
125
|
'name': 'RLIMIT_NOFILE',
|
126
126
|
})
|
@@ -130,11 +130,11 @@ class SupervisorSetupImpl(SupervisorSetup):
|
|
130
130
|
'msg': (
|
131
131
|
'The minimum number of available processes required to run this program is %(min_limit)s as per '
|
132
132
|
'the "minprocs" command-line argument or config file setting. The current environment will only '
|
133
|
-
'allow you to open %(hard)s processes.
|
133
|
+
'allow you to open %(hard)s processes. Either raise the number of usable processes in your '
|
134
134
|
'environment (see README.rst) or lower the minprocs setting in the config file to allow the '
|
135
135
|
'program to start.'
|
136
136
|
),
|
137
|
-
'min': self._config.
|
137
|
+
'min': self._config.min_procs,
|
138
138
|
'resource': resource.RLIMIT_NPROC,
|
139
139
|
'name': 'RLIMIT_NPROC',
|
140
140
|
})
|
@@ -220,11 +220,11 @@ class SupervisorSetupImpl(SupervisorSetup):
|
|
220
220
|
dl.after_daemonize()
|
221
221
|
|
222
222
|
def _do_daemonize(self) -> None:
|
223
|
-
# To daemonize, we need to become the leader of our own session (process) group.
|
224
|
-
# our parent process will also be sent to us.
|
223
|
+
# To daemonize, we need to become the leader of our own session (process) group. If we do not, signals sent to
|
224
|
+
# our parent process will also be sent to us. This might be bad because signals such as SIGINT can be sent to
|
225
225
|
# our parent process during normal (uninteresting) operations such as when we press Ctrl-C in the parent
|
226
226
|
# terminal window to escape from a logtail command. To disassociate ourselves from our parent's session group we
|
227
|
-
# use os.setsid.
|
227
|
+
# use os.setsid. It means "set session id", which has the effect of disassociating a process from is current
|
228
228
|
# session and process group and setting itself up as a new session leader.
|
229
229
|
#
|
230
230
|
# Unfortunately we cannot call setsid if we're already a session group leader, so we use "fork" to make a copy
|
@@ -256,7 +256,7 @@ class SupervisorSetupImpl(SupervisorSetup):
|
|
256
256
|
os.dup2(2, os.open('/dev/null', os.O_WRONLY))
|
257
257
|
|
258
258
|
# XXX Stevens, in his Advanced Unix book, section 13.3 (page 417) recommends calling umask(0) and closing unused
|
259
|
-
# file descriptors.
|
259
|
+
# file descriptors. In his Network Programming book, he additionally recommends ignoring SIGHUP and forking
|
260
260
|
# again after the setsid() call, for obscure SVR4 reasons.
|
261
261
|
os.setsid()
|
262
262
|
os.umask(self._config.umask)
|
@@ -316,7 +316,7 @@ class ProcessSpawningImpl(ProcessSpawning):
|
|
316
316
|
else:
|
317
317
|
os.dup2(check_not_none(pipes.child_stderr), 2)
|
318
318
|
|
319
|
-
for i in range(3, self._server_config.
|
319
|
+
for i in range(3, self._server_config.min_fds):
|
320
320
|
if i in self._inherited_fds:
|
321
321
|
continue
|
322
322
|
close_fd(Fd(i))
|
ominfra/supervisor/states.py
CHANGED
@@ -27,6 +27,17 @@ class ProcessState(enum.IntEnum):
|
|
27
27
|
return self in SIGNALABLE_STATES
|
28
28
|
|
29
29
|
|
30
|
+
# http://supervisord.org/subprocess.html
|
31
|
+
STATE_TRANSITIONS = {
|
32
|
+
ProcessState.STOPPED: (ProcessState.STARTING,),
|
33
|
+
ProcessState.STARTING: (ProcessState.RUNNING, ProcessState.BACKOFF, ProcessState.STOPPING),
|
34
|
+
ProcessState.RUNNING: (ProcessState.STOPPING, ProcessState.EXITED),
|
35
|
+
ProcessState.BACKOFF: (ProcessState.STARTING, ProcessState.FATAL),
|
36
|
+
ProcessState.STOPPING: (ProcessState.STOPPED,),
|
37
|
+
ProcessState.EXITED: (ProcessState.STARTING,),
|
38
|
+
ProcessState.FATAL: (ProcessState.STARTING,),
|
39
|
+
}
|
40
|
+
|
30
41
|
STOPPED_STATES = (
|
31
42
|
ProcessState.STOPPED,
|
32
43
|
ProcessState.EXITED,
|