ominfra 0.0.0.dev129__py3-none-any.whl → 0.0.0.dev131__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ominfra/deploy/_executor.py +16 -1
- ominfra/deploy/poly/_main.py +2 -2
- ominfra/pyremote/_runcommands.py +16 -1
- ominfra/scripts/journald2aws.py +117 -5
- ominfra/scripts/supervisor.py +1450 -703
- ominfra/supervisor/configs.py +17 -17
- ominfra/supervisor/dispatchers.py +7 -6
- ominfra/supervisor/dispatchersimpl.py +11 -15
- ominfra/supervisor/groups.py +16 -1
- ominfra/supervisor/http.py +130 -0
- ominfra/supervisor/inject.py +44 -5
- ominfra/supervisor/io.py +39 -24
- ominfra/supervisor/pipes.py +2 -2
- ominfra/supervisor/privileges.py +4 -6
- ominfra/supervisor/processimpl.py +33 -34
- ominfra/supervisor/setupimpl.py +16 -16
- ominfra/supervisor/spawningimpl.py +3 -3
- ominfra/supervisor/supervisor.py +6 -3
- ominfra/supervisor/types.py +6 -56
- ominfra/supervisor/utils/os.py +1 -1
- ominfra/supervisor/utils/strings.py +2 -2
- {ominfra-0.0.0.dev129.dist-info → ominfra-0.0.0.dev131.dist-info}/METADATA +3 -3
- {ominfra-0.0.0.dev129.dist-info → ominfra-0.0.0.dev131.dist-info}/RECORD +27 -27
- ominfra/supervisor/poller.py +0 -240
- {ominfra-0.0.0.dev129.dist-info → ominfra-0.0.0.dev131.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev129.dist-info → ominfra-0.0.0.dev131.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev129.dist-info → ominfra-0.0.0.dev131.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev129.dist-info → ominfra-0.0.0.dev131.dist-info}/top_level.txt +0 -0
ominfra/supervisor/configs.py
CHANGED
@@ -37,32 +37,32 @@ class ProcessConfig:
|
|
37
37
|
umask: ta.Optional[int] = None
|
38
38
|
priority: int = 999
|
39
39
|
|
40
|
-
|
41
|
-
|
40
|
+
auto_start: bool = True
|
41
|
+
auto_restart: str = 'unexpected'
|
42
42
|
|
43
|
-
|
44
|
-
|
43
|
+
start_secs: int = 1
|
44
|
+
start_retries: int = 3
|
45
45
|
|
46
|
-
|
47
|
-
|
46
|
+
num_procs: int = 1
|
47
|
+
num_procs_start: int = 0
|
48
48
|
|
49
49
|
@dc.dataclass(frozen=True)
|
50
50
|
class Log:
|
51
51
|
file: ta.Optional[str] = None
|
52
|
-
|
52
|
+
capture_max_bytes: ta.Optional[int] = None
|
53
53
|
events_enabled: bool = False
|
54
54
|
syslog: bool = False
|
55
55
|
backups: ta.Optional[int] = None
|
56
|
-
|
56
|
+
max_bytes: ta.Optional[int] = None
|
57
57
|
|
58
58
|
stdout: Log = Log()
|
59
59
|
stderr: Log = Log()
|
60
60
|
|
61
|
-
|
62
|
-
|
63
|
-
|
61
|
+
stop_signal: int = signal.SIGTERM
|
62
|
+
stop_wait_secs: int = 10
|
63
|
+
stop_as_group: bool = False
|
64
64
|
|
65
|
-
|
65
|
+
kill_as_group: bool = False
|
66
66
|
|
67
67
|
exitcodes: ta.Sequence[int] = (0,)
|
68
68
|
|
@@ -87,14 +87,14 @@ class ServerConfig:
|
|
87
87
|
umask: int = 0o22
|
88
88
|
directory: ta.Optional[str] = None
|
89
89
|
logfile: str = 'supervisord.log'
|
90
|
-
|
90
|
+
logfile_max_bytes: int = 50 * 1024 * 1024
|
91
91
|
logfile_backups: int = 10
|
92
92
|
loglevel: int = logging.INFO
|
93
93
|
pidfile: str = 'supervisord.pid'
|
94
94
|
identifier: str = 'supervisor'
|
95
95
|
child_logdir: str = '/dev/null'
|
96
|
-
|
97
|
-
|
96
|
+
min_fds: int = 1024
|
97
|
+
min_procs: int = 200
|
98
98
|
nocleanup: bool = False
|
99
99
|
strip_ansi: bool = False
|
100
100
|
silent: bool = False
|
@@ -107,7 +107,7 @@ class ServerConfig:
|
|
107
107
|
umask: ta.Union[int, str] = 0o22,
|
108
108
|
directory: ta.Optional[str] = None,
|
109
109
|
logfile: str = 'supervisord.log',
|
110
|
-
|
110
|
+
logfile_max_bytes: ta.Union[int, str] = 50 * 1024 * 1024,
|
111
111
|
loglevel: ta.Union[int, str] = logging.INFO,
|
112
112
|
pidfile: str = 'supervisord.pid',
|
113
113
|
child_logdir: ta.Optional[str] = None,
|
@@ -117,7 +117,7 @@ class ServerConfig:
|
|
117
117
|
umask=parse_octal(umask),
|
118
118
|
directory=check_existing_dir(directory) if directory is not None else None,
|
119
119
|
logfile=check_path_with_existing_dir(logfile),
|
120
|
-
|
120
|
+
logfile_max_bytes=parse_bytes_size(logfile_max_bytes),
|
121
121
|
loglevel=parse_logging_level(loglevel),
|
122
122
|
pidfile=check_path_with_existing_dir(pidfile),
|
123
123
|
child_logdir=child_logdir if child_logdir else tempfile.gettempdir(),
|
@@ -1,13 +1,14 @@
|
|
1
1
|
# ruff: noqa: UP006 UP007
|
2
|
-
from .
|
2
|
+
from omlish.lite.fdio.handlers import FdIoHandler
|
3
|
+
|
3
4
|
from .types import ProcessOutputDispatcher
|
4
5
|
from .utils.collections import KeyedCollection
|
5
6
|
from .utils.ostypes import Fd
|
6
7
|
|
7
8
|
|
8
|
-
class Dispatchers(KeyedCollection[Fd,
|
9
|
-
def _key(self, v:
|
10
|
-
return v.fd
|
9
|
+
class Dispatchers(KeyedCollection[Fd, FdIoHandler]):
|
10
|
+
def _key(self, v: FdIoHandler) -> Fd:
|
11
|
+
return Fd(v.fd())
|
11
12
|
|
12
13
|
#
|
13
14
|
|
@@ -16,9 +17,9 @@ class Dispatchers(KeyedCollection[Fd, Dispatcher]):
|
|
16
17
|
# note that we *must* call readable() for every dispatcher, as it may have side effects for a given
|
17
18
|
# dispatcher (eg. call handle_listener_state_change for event listener processes)
|
18
19
|
if d.readable():
|
19
|
-
d.
|
20
|
+
d.on_readable()
|
20
21
|
if d.writable():
|
21
|
-
d.
|
22
|
+
d.on_writable()
|
22
23
|
|
23
24
|
#
|
24
25
|
|
@@ -60,7 +60,6 @@ class BaseProcessDispatcherImpl(ProcessDispatcher, abc.ABC):
|
|
60
60
|
def channel(self) -> str:
|
61
61
|
return self._channel
|
62
62
|
|
63
|
-
@property
|
64
63
|
def fd(self) -> Fd:
|
65
64
|
return self._fd
|
66
65
|
|
@@ -75,7 +74,7 @@ class BaseProcessDispatcherImpl(ProcessDispatcher, abc.ABC):
|
|
75
74
|
log.debug('fd %s closed, stopped monitoring %s', self._fd, self)
|
76
75
|
self._closed = True
|
77
76
|
|
78
|
-
def
|
77
|
+
def on_error(self, exc: ta.Optional[BaseException] = None) -> None:
|
79
78
|
nil, t, v, tbinfo = compact_traceback()
|
80
79
|
|
81
80
|
log.critical('uncaptured python exception, closing channel %s (%s:%s %s)', repr(self), t, v, tbinfo)
|
@@ -148,7 +147,7 @@ class ProcessOutputDispatcherImpl(BaseProcessDispatcherImpl, ProcessOutputDispat
|
|
148
147
|
channel = self._channel # noqa
|
149
148
|
|
150
149
|
logfile = self._lc.file
|
151
|
-
|
150
|
+
max_bytes = self._lc.max_bytes # noqa
|
152
151
|
backups = self._lc.backups # noqa
|
153
152
|
to_syslog = self._lc.syslog
|
154
153
|
|
@@ -160,8 +159,8 @@ class ProcessOutputDispatcherImpl(BaseProcessDispatcherImpl, ProcessOutputDispat
|
|
160
159
|
# self.normal_log,
|
161
160
|
# filename=logfile,
|
162
161
|
# fmt='%(message)s',
|
163
|
-
# rotating=bool(
|
164
|
-
#
|
162
|
+
# rotating=bool(max_bytes), # optimization
|
163
|
+
# max_bytes=max_bytes,
|
165
164
|
# backups=backups,
|
166
165
|
# )
|
167
166
|
|
@@ -173,17 +172,17 @@ class ProcessOutputDispatcherImpl(BaseProcessDispatcherImpl, ProcessOutputDispat
|
|
173
172
|
|
174
173
|
def _init_capture_log(self) -> None:
|
175
174
|
"""
|
176
|
-
Configure the capture log for this process.
|
175
|
+
Configure the capture log for this process. This log is used to temporarily capture output when special output
|
177
176
|
is detected. Sets self.capture_log if capturing is enabled.
|
178
177
|
"""
|
179
178
|
|
180
|
-
|
181
|
-
if
|
179
|
+
capture_max_bytes = self._lc.capture_max_bytes
|
180
|
+
if capture_max_bytes:
|
182
181
|
self._capture_log = logging.getLogger(__name__)
|
183
182
|
# loggers.handle_boundIO(
|
184
183
|
# self._capture_log,
|
185
184
|
# fmt='%(message)s',
|
186
|
-
#
|
185
|
+
# max_bytes=capture_max_bytes,
|
187
186
|
# )
|
188
187
|
|
189
188
|
def remove_logs(self) -> None:
|
@@ -291,12 +290,12 @@ class ProcessOutputDispatcherImpl(BaseProcessDispatcherImpl, ProcessOutputDispat
|
|
291
290
|
return False
|
292
291
|
return True
|
293
292
|
|
294
|
-
def
|
293
|
+
def on_readable(self) -> None:
|
295
294
|
data = read_fd(self._fd)
|
296
295
|
self._output_buffer += data
|
297
296
|
self.record_output()
|
298
297
|
if not data:
|
299
|
-
# if we get no data back from the pipe, it means that the child process has ended.
|
298
|
+
# if we get no data back from the pipe, it means that the child process has ended. See
|
300
299
|
# mail.python.org/pipermail/python-dev/2004-August/046850.html
|
301
300
|
self.close()
|
302
301
|
|
@@ -329,15 +328,12 @@ class ProcessInputDispatcherImpl(BaseProcessDispatcherImpl, ProcessInputDispatch
|
|
329
328
|
return True
|
330
329
|
return False
|
331
330
|
|
332
|
-
def readable(self) -> bool:
|
333
|
-
return False
|
334
|
-
|
335
331
|
def flush(self) -> None:
|
336
332
|
# other code depends on this raising EPIPE if the pipe is closed
|
337
333
|
sent = os.write(self._fd, as_bytes(self._input_buffer))
|
338
334
|
self._input_buffer = self._input_buffer[sent:]
|
339
335
|
|
340
|
-
def
|
336
|
+
def on_writable(self) -> None:
|
341
337
|
if self._input_buffer:
|
342
338
|
try:
|
343
339
|
self.flush()
|
ominfra/supervisor/groups.py
CHANGED
@@ -2,15 +2,20 @@
|
|
2
2
|
import typing as ta
|
3
3
|
|
4
4
|
from .configs import ProcessGroupConfig
|
5
|
+
from .dispatchers import Dispatchers
|
5
6
|
from .events import EventCallbacks
|
6
7
|
from .events import ProcessGroupAddedEvent
|
7
8
|
from .events import ProcessGroupRemovedEvent
|
9
|
+
from .types import HasDispatchers
|
8
10
|
from .types import Process
|
9
11
|
from .types import ProcessGroup
|
10
12
|
from .utils.collections import KeyedCollectionAccessors
|
11
13
|
|
12
14
|
|
13
|
-
class ProcessGroupManager(
|
15
|
+
class ProcessGroupManager(
|
16
|
+
KeyedCollectionAccessors[str, ProcessGroup],
|
17
|
+
HasDispatchers,
|
18
|
+
):
|
14
19
|
def __init__(
|
15
20
|
self,
|
16
21
|
*,
|
@@ -34,6 +39,16 @@ class ProcessGroupManager(KeyedCollectionAccessors[str, ProcessGroup]):
|
|
34
39
|
|
35
40
|
#
|
36
41
|
|
42
|
+
def get_dispatchers(self) -> Dispatchers:
|
43
|
+
return Dispatchers(
|
44
|
+
d
|
45
|
+
for g in self
|
46
|
+
for p in g
|
47
|
+
for d in p.get_dispatchers()
|
48
|
+
)
|
49
|
+
|
50
|
+
#
|
51
|
+
|
37
52
|
def add(self, group: ProcessGroup) -> None:
|
38
53
|
if (name := group.name) in self._by_name:
|
39
54
|
raise KeyError(f'Process group already exists: {name}')
|
@@ -0,0 +1,130 @@
|
|
1
|
+
# ruff: noqa: U006 UP007
|
2
|
+
import json
|
3
|
+
import socket
|
4
|
+
import typing as ta
|
5
|
+
|
6
|
+
from omlish.lite.check import check_not_none
|
7
|
+
from omlish.lite.fdio.corohttp import CoroHttpServerConnectionFdIoHandler
|
8
|
+
from omlish.lite.fdio.handlers import SocketFdIoHandler
|
9
|
+
from omlish.lite.http.handlers import HttpHandler
|
10
|
+
from omlish.lite.http.handlers import HttpHandlerRequest
|
11
|
+
from omlish.lite.http.handlers import HttpHandlerResponse
|
12
|
+
from omlish.lite.json import JSON_PRETTY_KWARGS
|
13
|
+
from omlish.lite.socket import SocketAddress
|
14
|
+
|
15
|
+
from .dispatchers import Dispatchers
|
16
|
+
from .groups import ProcessGroupManager
|
17
|
+
from .types import HasDispatchers
|
18
|
+
|
19
|
+
|
20
|
+
##
|
21
|
+
|
22
|
+
|
23
|
+
class SocketServerFdIoHandler(SocketFdIoHandler):
|
24
|
+
def __init__(
|
25
|
+
self,
|
26
|
+
addr: SocketAddress,
|
27
|
+
on_connect: ta.Callable[[socket.socket, SocketAddress], None],
|
28
|
+
) -> None:
|
29
|
+
sock = socket.create_server(addr)
|
30
|
+
sock.setblocking(False)
|
31
|
+
|
32
|
+
super().__init__(addr, sock)
|
33
|
+
|
34
|
+
self._on_connect = on_connect
|
35
|
+
|
36
|
+
sock.listen(1)
|
37
|
+
|
38
|
+
def readable(self) -> bool:
|
39
|
+
return True
|
40
|
+
|
41
|
+
def on_readable(self) -> None:
|
42
|
+
cli_sock, cli_addr = check_not_none(self._sock).accept()
|
43
|
+
cli_sock.setblocking(False)
|
44
|
+
|
45
|
+
self._on_connect(cli_sock, cli_addr)
|
46
|
+
|
47
|
+
|
48
|
+
##
|
49
|
+
|
50
|
+
|
51
|
+
class HttpServer(HasDispatchers):
|
52
|
+
class Address(ta.NamedTuple):
|
53
|
+
a: SocketAddress
|
54
|
+
|
55
|
+
class Handler(ta.NamedTuple):
|
56
|
+
h: HttpHandler
|
57
|
+
|
58
|
+
def __init__(
|
59
|
+
self,
|
60
|
+
handler: Handler,
|
61
|
+
addr: Address = Address(('localhost', 8000)),
|
62
|
+
) -> None:
|
63
|
+
super().__init__()
|
64
|
+
|
65
|
+
self._handler = handler.h
|
66
|
+
self._addr = addr.a
|
67
|
+
|
68
|
+
self._server = SocketServerFdIoHandler(self._addr, self._on_connect)
|
69
|
+
|
70
|
+
self._conns: ta.List[CoroHttpServerConnectionFdIoHandler] = []
|
71
|
+
|
72
|
+
def get_dispatchers(self) -> Dispatchers:
|
73
|
+
l = []
|
74
|
+
for c in self._conns:
|
75
|
+
if not c.closed:
|
76
|
+
l.append(c)
|
77
|
+
self._conns = l
|
78
|
+
return Dispatchers([
|
79
|
+
self._server,
|
80
|
+
*l,
|
81
|
+
])
|
82
|
+
|
83
|
+
def _on_connect(self, sock: socket.socket, addr: SocketAddress) -> None:
|
84
|
+
conn = CoroHttpServerConnectionFdIoHandler(
|
85
|
+
addr,
|
86
|
+
sock,
|
87
|
+
self._handler,
|
88
|
+
)
|
89
|
+
|
90
|
+
self._conns.append(conn)
|
91
|
+
|
92
|
+
|
93
|
+
##
|
94
|
+
|
95
|
+
|
96
|
+
class SupervisorHttpHandler:
|
97
|
+
def __init__(
|
98
|
+
self,
|
99
|
+
*,
|
100
|
+
groups: ProcessGroupManager,
|
101
|
+
) -> None:
|
102
|
+
super().__init__()
|
103
|
+
|
104
|
+
self._groups = groups
|
105
|
+
|
106
|
+
def handle(self, req: HttpHandlerRequest) -> HttpHandlerResponse:
|
107
|
+
dct = {
|
108
|
+
'method': req.method,
|
109
|
+
'path': req.path,
|
110
|
+
'data': len(req.data or b''),
|
111
|
+
'groups': {
|
112
|
+
g.name: {
|
113
|
+
'processes': {
|
114
|
+
p.name: {
|
115
|
+
'pid': p.pid,
|
116
|
+
}
|
117
|
+
for p in g
|
118
|
+
},
|
119
|
+
}
|
120
|
+
for g in self._groups
|
121
|
+
},
|
122
|
+
}
|
123
|
+
|
124
|
+
return HttpHandlerResponse(
|
125
|
+
200,
|
126
|
+
data=json.dumps(dct, **JSON_PRETTY_KWARGS).encode('utf-8') + b'\n',
|
127
|
+
headers={
|
128
|
+
'Content-Type': 'application/json',
|
129
|
+
},
|
130
|
+
)
|
ominfra/supervisor/inject.py
CHANGED
@@ -1,6 +1,11 @@
|
|
1
1
|
# ruff: noqa: UP006 UP007
|
2
|
+
import dataclasses as dc
|
2
3
|
import typing as ta
|
3
4
|
|
5
|
+
from omlish.lite.fdio.kqueue import KqueueFdIoPoller # noqa
|
6
|
+
from omlish.lite.fdio.pollers import FdIoPoller
|
7
|
+
from omlish.lite.fdio.pollers import PollFdIoPoller # noqa
|
8
|
+
from omlish.lite.fdio.pollers import SelectFdIoPoller
|
4
9
|
from omlish.lite.inject import InjectorBindingOrBindings
|
5
10
|
from omlish.lite.inject import InjectorBindings
|
6
11
|
from omlish.lite.inject import inj
|
@@ -12,9 +17,10 @@ from .events import EventCallbacks
|
|
12
17
|
from .groups import ProcessGroupManager
|
13
18
|
from .groupsimpl import ProcessFactory
|
14
19
|
from .groupsimpl import ProcessGroupImpl
|
20
|
+
from .http import HttpServer
|
21
|
+
from .http import SupervisorHttpHandler
|
22
|
+
from .io import HasDispatchersList
|
15
23
|
from .io import IoManager
|
16
|
-
from .poller import Poller
|
17
|
-
from .poller import get_poller_impl
|
18
24
|
from .process import PidHistory
|
19
25
|
from .processimpl import ProcessImpl
|
20
26
|
from .processimpl import ProcessSpawningFactory
|
@@ -31,12 +37,24 @@ from .spawningimpl import ProcessSpawningImpl
|
|
31
37
|
from .supervisor import ProcessGroupFactory
|
32
38
|
from .supervisor import Supervisor
|
33
39
|
from .supervisor import SupervisorStateManagerImpl
|
40
|
+
from .types import HasDispatchers
|
34
41
|
from .types import ServerEpoch
|
35
42
|
from .types import SupervisorStateManager
|
36
43
|
from .utils.signals import SignalReceiver
|
37
44
|
from .utils.users import get_user
|
38
45
|
|
39
46
|
|
47
|
+
@dc.dataclass(frozen=True)
|
48
|
+
class _FdIoPollerDaemonizeListener(DaemonizeListener):
|
49
|
+
_poller: FdIoPoller
|
50
|
+
|
51
|
+
def before_daemonize(self) -> None:
|
52
|
+
self._poller.close()
|
53
|
+
|
54
|
+
def after_daemonize(self) -> None:
|
55
|
+
self._poller.reopen()
|
56
|
+
|
57
|
+
|
40
58
|
def bind_server(
|
41
59
|
config: ServerConfig,
|
42
60
|
*,
|
@@ -46,22 +64,24 @@ def bind_server(
|
|
46
64
|
lst: ta.List[InjectorBindingOrBindings] = [
|
47
65
|
inj.bind(config),
|
48
66
|
|
67
|
+
inj.bind_array(DaemonizeListener),
|
49
68
|
inj.bind_array_type(DaemonizeListener, DaemonizeListeners),
|
50
69
|
|
51
70
|
inj.bind(SupervisorSetupImpl, singleton=True),
|
52
71
|
inj.bind(SupervisorSetup, to_key=SupervisorSetupImpl),
|
53
72
|
|
54
|
-
inj.bind(DaemonizeListener, array=True, to_key=Poller),
|
55
|
-
|
56
73
|
inj.bind(EventCallbacks, singleton=True),
|
57
74
|
|
58
75
|
inj.bind(SignalReceiver, singleton=True),
|
59
76
|
|
60
77
|
inj.bind(IoManager, singleton=True),
|
78
|
+
inj.bind_array(HasDispatchers),
|
79
|
+
inj.bind_array_type(HasDispatchers, HasDispatchersList),
|
61
80
|
|
62
81
|
inj.bind(SignalHandler, singleton=True),
|
63
82
|
|
64
83
|
inj.bind(ProcessGroupManager, singleton=True),
|
84
|
+
inj.bind(HasDispatchers, array=True, to_key=ProcessGroupManager),
|
65
85
|
|
66
86
|
inj.bind(Supervisor, singleton=True),
|
67
87
|
|
@@ -94,7 +114,26 @@ def bind_server(
|
|
94
114
|
|
95
115
|
#
|
96
116
|
|
97
|
-
|
117
|
+
poller_impl = next(filter(None, [
|
118
|
+
KqueueFdIoPoller,
|
119
|
+
PollFdIoPoller,
|
120
|
+
SelectFdIoPoller,
|
121
|
+
]))
|
122
|
+
lst.append(inj.bind(poller_impl, key=FdIoPoller, singleton=True))
|
123
|
+
inj.bind(_FdIoPollerDaemonizeListener, array=True, singleton=True)
|
124
|
+
|
125
|
+
#
|
126
|
+
|
127
|
+
def _provide_http_handler(s: SupervisorHttpHandler) -> HttpServer.Handler:
|
128
|
+
return HttpServer.Handler(s.handle)
|
129
|
+
|
130
|
+
lst.extend([
|
131
|
+
inj.bind(HttpServer, singleton=True, eager=True),
|
132
|
+
inj.bind(HasDispatchers, array=True, to_key=HttpServer),
|
133
|
+
|
134
|
+
inj.bind(SupervisorHttpHandler, singleton=True),
|
135
|
+
inj.bind(_provide_http_handler),
|
136
|
+
])
|
98
137
|
|
99
138
|
#
|
100
139
|
|
ominfra/supervisor/io.py
CHANGED
@@ -1,58 +1,71 @@
|
|
1
1
|
# ruff: noqa: UP006 UP007
|
2
|
+
import typing as ta
|
3
|
+
|
4
|
+
from omlish.lite.fdio.pollers import FdIoPoller
|
2
5
|
from omlish.lite.logs import log
|
3
6
|
|
4
7
|
from .dispatchers import Dispatchers
|
5
|
-
from .groups import ProcessGroupManager
|
6
|
-
from .poller import Poller
|
7
8
|
from .types import ExitNow
|
9
|
+
from .types import HasDispatchers
|
10
|
+
from .utils.ostypes import Fd
|
8
11
|
|
9
12
|
|
10
13
|
##
|
11
14
|
|
12
15
|
|
13
|
-
|
16
|
+
HasDispatchersList = ta.NewType('HasDispatchersList', ta.Sequence[HasDispatchers])
|
17
|
+
|
18
|
+
|
19
|
+
class IoManager(HasDispatchers):
|
14
20
|
def __init__(
|
15
21
|
self,
|
16
22
|
*,
|
17
|
-
poller:
|
18
|
-
|
23
|
+
poller: FdIoPoller,
|
24
|
+
has_dispatchers_list: HasDispatchersList,
|
19
25
|
) -> None:
|
20
26
|
super().__init__()
|
21
27
|
|
22
28
|
self._poller = poller
|
23
|
-
self.
|
29
|
+
self._has_dispatchers_list = has_dispatchers_list
|
24
30
|
|
25
31
|
def get_dispatchers(self) -> Dispatchers:
|
26
32
|
return Dispatchers(
|
27
33
|
d
|
28
|
-
for
|
29
|
-
for d in
|
34
|
+
for hd in self._has_dispatchers_list
|
35
|
+
for d in hd.get_dispatchers()
|
30
36
|
)
|
31
37
|
|
32
38
|
def poll(self) -> None:
|
33
39
|
dispatchers = self.get_dispatchers()
|
34
40
|
|
35
|
-
|
36
|
-
if
|
37
|
-
|
38
|
-
|
39
|
-
self._poller.register_writable(fd)
|
41
|
+
self._poller.update(
|
42
|
+
{fd for fd, d in dispatchers.items() if d.readable()},
|
43
|
+
{fd for fd, d in dispatchers.items() if d.writable()},
|
44
|
+
)
|
40
45
|
|
41
46
|
timeout = 1 # this cannot be fewer than the smallest TickEvent (5)
|
42
|
-
|
47
|
+
log.info(f'Polling: {timeout=}') # noqa
|
48
|
+
polled = self._poller.poll(timeout)
|
49
|
+
log.info(f'Polled: {polled=}') # noqa
|
50
|
+
if polled.msg is not None:
|
51
|
+
log.error(polled.msg)
|
52
|
+
if polled.exc is not None:
|
53
|
+
log.error('Poll exception: %r', polled.exc)
|
43
54
|
|
44
|
-
for
|
55
|
+
for r in polled.r:
|
56
|
+
fd = Fd(r)
|
45
57
|
if fd in dispatchers:
|
58
|
+
dispatcher = dispatchers[fd]
|
46
59
|
try:
|
47
|
-
dispatcher = dispatchers[fd]
|
48
60
|
log.debug('read event caused by %r', dispatcher)
|
49
|
-
dispatcher.
|
61
|
+
dispatcher.on_readable()
|
50
62
|
if not dispatcher.readable():
|
51
63
|
self._poller.unregister_readable(fd)
|
52
64
|
except ExitNow:
|
53
65
|
raise
|
54
|
-
except Exception: # noqa
|
55
|
-
|
66
|
+
except Exception as exc: # noqa
|
67
|
+
log.exception('Error in dispatcher: %r', dispatcher)
|
68
|
+
dispatcher.on_error(exc)
|
56
69
|
else:
|
57
70
|
# if the fd is not in combined map, we should unregister it. otherwise, it will be polled every
|
58
71
|
# time, which may cause 100% cpu usage
|
@@ -62,18 +75,20 @@ class IoManager:
|
|
62
75
|
except Exception: # noqa
|
63
76
|
pass
|
64
77
|
|
65
|
-
for
|
78
|
+
for w in polled.w:
|
79
|
+
fd = Fd(w)
|
66
80
|
if fd in dispatchers:
|
81
|
+
dispatcher = dispatchers[fd]
|
67
82
|
try:
|
68
|
-
dispatcher = dispatchers[fd]
|
69
83
|
log.debug('write event caused by %r', dispatcher)
|
70
|
-
dispatcher.
|
84
|
+
dispatcher.on_writable()
|
71
85
|
if not dispatcher.writable():
|
72
86
|
self._poller.unregister_writable(fd)
|
73
87
|
except ExitNow:
|
74
88
|
raise
|
75
|
-
except Exception: # noqa
|
76
|
-
|
89
|
+
except Exception as exc: # noqa
|
90
|
+
log.exception('Error in dispatcher: %r', dispatcher)
|
91
|
+
dispatcher.on_error(exc)
|
77
92
|
else:
|
78
93
|
log.debug('unexpected write event from fd %r', fd)
|
79
94
|
try:
|
ominfra/supervisor/pipes.py
CHANGED
@@ -29,8 +29,8 @@ class ProcessPipes:
|
|
29
29
|
|
30
30
|
def make_process_pipes(stderr=True) -> ProcessPipes:
|
31
31
|
"""
|
32
|
-
Create pipes for parent to child stdin/stdout/stderr communications.
|
33
|
-
|
32
|
+
Create pipes for parent to child stdin/stdout/stderr communications. Open fd in non-blocking mode so we can read
|
33
|
+
them in the mainloop without blocking. If stderr is False, don't create a pipe for stderr.
|
34
34
|
"""
|
35
35
|
|
36
36
|
pipes: ta.Dict[str, ta.Optional[Fd]] = {
|
ominfra/supervisor/privileges.py
CHANGED
@@ -7,9 +7,8 @@ import typing as ta
|
|
7
7
|
|
8
8
|
def drop_privileges(user: ta.Union[int, str, None]) -> ta.Optional[str]:
|
9
9
|
"""
|
10
|
-
Drop privileges to become the specified user, which may be a username or uid.
|
11
|
-
|
12
|
-
dropped.
|
10
|
+
Drop privileges to become the specified user, which may be a username or uid. Called for supervisord startup and
|
11
|
+
when spawning subprocesses. Returns None on success or a string error message if privileges could not be dropped.
|
13
12
|
"""
|
14
13
|
|
15
14
|
if user is None:
|
@@ -33,9 +32,8 @@ def drop_privileges(user: ta.Union[int, str, None]) -> ta.Optional[str]:
|
|
33
32
|
current_uid = os.getuid()
|
34
33
|
|
35
34
|
if current_uid == uid:
|
36
|
-
# do nothing and return successfully if the uid is already the current one.
|
37
|
-
#
|
38
|
-
# it.
|
35
|
+
# do nothing and return successfully if the uid is already the current one. this allows a supervisord running as
|
36
|
+
# an unprivileged user "foo" to start a process where the config has "user=foo" (same user) in it.
|
39
37
|
return None
|
40
38
|
|
41
39
|
if current_uid != 0:
|