ominfra 0.0.0.dev117__py3-none-any.whl → 0.0.0.dev119__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,4 @@
1
1
  if __name__ == '__main__':
2
- from .supervisor import main
2
+ from .main import main
3
3
 
4
4
  main()
@@ -2,7 +2,6 @@
2
2
  import errno
3
3
  import fcntl
4
4
  import grp
5
- import logging
6
5
  import os
7
6
  import pwd
8
7
  import re
@@ -12,6 +11,8 @@ import stat
12
11
  import typing as ta
13
12
  import warnings
14
13
 
14
+ from omlish.lite.logs import log
15
+
15
16
  from .compat import SignalReceiver
16
17
  from .compat import close_fd
17
18
  from .compat import mktempfile
@@ -23,6 +24,7 @@ from .datatypes import name_to_uid
23
24
  from .exceptions import NoPermissionError
24
25
  from .exceptions import NotExecutableError
25
26
  from .exceptions import NotFoundError
27
+ from .poller import BasePoller
26
28
  from .poller import Poller
27
29
  from .states import SupervisorState
28
30
  from .states import SupervisorStates
@@ -30,41 +32,47 @@ from .types import AbstractServerContext
30
32
  from .types import AbstractSubprocess
31
33
 
32
34
 
33
- log = logging.getLogger(__name__)
34
-
35
-
36
35
  class ServerContext(AbstractServerContext):
37
- first = False
38
- test = False
39
-
40
- ##
41
-
42
- def __init__(self, config: ServerConfig) -> None:
36
+ def __init__(
37
+ self,
38
+ config: ServerConfig,
39
+ *,
40
+ epoch: int = 0,
41
+ ) -> None:
43
42
  super().__init__()
44
43
 
45
44
  self._config = config
45
+ self._epoch = epoch
46
46
 
47
47
  self._pid_history: ta.Dict[int, AbstractSubprocess] = {}
48
48
  self._state: SupervisorState = SupervisorStates.RUNNING
49
49
 
50
- self.signal_receiver = SignalReceiver()
50
+ self._signal_receiver = SignalReceiver()
51
51
 
52
- self.poller = Poller()
52
+ self._poller: BasePoller = Poller()
53
53
 
54
- if self.config.user is not None:
55
- uid = name_to_uid(self.config.user)
56
- self.uid = uid
57
- self.gid = gid_for_uid(uid)
54
+ if config.user is not None:
55
+ uid = name_to_uid(config.user)
56
+ self._uid: ta.Optional[int] = uid
57
+ self._gid: ta.Optional[int] = gid_for_uid(uid)
58
58
  else:
59
- self.uid = None
60
- self.gid = None
59
+ self._uid = None
60
+ self._gid = None
61
61
 
62
- self.unlink_pidfile = False
62
+ self._unlink_pidfile = False
63
63
 
64
64
  @property
65
65
  def config(self) -> ServerConfig:
66
66
  return self._config
67
67
 
68
+ @property
69
+ def epoch(self) -> int:
70
+ return self._epoch
71
+
72
+ @property
73
+ def first(self) -> bool:
74
+ return not self._epoch
75
+
68
76
  @property
69
77
  def state(self) -> SupervisorState:
70
78
  return self._state
@@ -72,17 +80,26 @@ class ServerContext(AbstractServerContext):
72
80
  def set_state(self, state: SupervisorState) -> None:
73
81
  self._state = state
74
82
 
83
+ @property
84
+ def poller(self) -> BasePoller:
85
+ return self._poller
86
+
75
87
  @property
76
88
  def pid_history(self) -> ta.Dict[int, AbstractSubprocess]:
77
89
  return self._pid_history
78
90
 
79
- uid: ta.Optional[int]
80
- gid: ta.Optional[int]
91
+ @property
92
+ def uid(self) -> ta.Optional[int]:
93
+ return self._uid
94
+
95
+ @property
96
+ def gid(self) -> ta.Optional[int]:
97
+ return self._gid
81
98
 
82
99
  ##
83
100
 
84
101
  def set_signals(self) -> None:
85
- self.signal_receiver.install(
102
+ self._signal_receiver.install(
86
103
  signal.SIGTERM,
87
104
  signal.SIGINT,
88
105
  signal.SIGQUIT,
@@ -193,7 +210,7 @@ class ServerContext(AbstractServerContext):
193
210
  ))
194
211
 
195
212
  def cleanup(self) -> None:
196
- if self.unlink_pidfile:
213
+ if self._unlink_pidfile:
197
214
  try_unlink(self.config.pidfile)
198
215
  self.poller.close()
199
216
 
@@ -246,6 +263,7 @@ class ServerContext(AbstractServerContext):
246
263
  # Parent
247
264
  log.debug('supervisord forked; parent exiting')
248
265
  real_exit(0)
266
+
249
267
  # Child
250
268
  log.info('daemonizing the supervisord process')
251
269
  if self.config.directory:
@@ -255,11 +273,15 @@ class ServerContext(AbstractServerContext):
255
273
  log.critical("can't chdir into %r: %s", self.config.directory, err)
256
274
  else:
257
275
  log.info('set current directory: %r', self.config.directory)
276
+
258
277
  os.dup2(0, os.open('/dev/null', os.O_RDONLY))
259
278
  os.dup2(1, os.open('/dev/null', os.O_WRONLY))
260
279
  os.dup2(2, os.open('/dev/null', os.O_WRONLY))
280
+
261
281
  os.setsid()
282
+
262
283
  os.umask(self.config.umask)
284
+
263
285
  # XXX Stevens, in his Advanced Unix book, section 13.3 (page 417) recommends calling umask(0) and closing unused
264
286
  # file descriptors. In his Network Programming book, he additionally recommends ignoring SIGHUP and forking
265
287
  # again after the setsid() call, for obscure SVR4 reasons.
@@ -274,7 +296,7 @@ class ServerContext(AbstractServerContext):
274
296
  return logfile
275
297
 
276
298
  def get_signal(self) -> ta.Optional[int]:
277
- return self.signal_receiver.get_signal()
299
+ return self._signal_receiver.get_signal()
278
300
 
279
301
  def write_pidfile(self) -> None:
280
302
  pid = os.getpid()
@@ -284,7 +306,7 @@ class ServerContext(AbstractServerContext):
284
306
  except OSError:
285
307
  log.critical('could not write pidfile %s', self.config.pidfile)
286
308
  else:
287
- self.unlink_pidfile = True
309
+ self._unlink_pidfile = True
288
310
  log.info('supervisord started with pid %s', pid)
289
311
 
290
312
 
@@ -337,11 +359,14 @@ def drop_privileges(user: ta.Union[int, str, None]) -> ta.Optional[str]:
337
359
  os.setgroups(groups)
338
360
  except OSError:
339
361
  return 'Could not set groups of effective user'
362
+
340
363
  try:
341
364
  os.setgid(gid)
342
365
  except OSError:
343
366
  return 'Could not set group id of effective user'
367
+
344
368
  os.setuid(uid)
369
+
345
370
  return None
346
371
 
347
372
 
@@ -5,6 +5,8 @@ import logging
5
5
  import os
6
6
  import typing as ta
7
7
 
8
+ from omlish.lite.logs import log
9
+
8
10
  from .compat import as_bytes
9
11
  from .compat import compact_traceback
10
12
  from .compat import find_prefix_at_end
@@ -17,9 +19,6 @@ from .events import notify_event
17
19
  from .types import AbstractSubprocess
18
20
 
19
21
 
20
- log = logging.getLogger(__name__)
21
-
22
-
23
22
  class Dispatcher(abc.ABC):
24
23
 
25
24
  def __init__(self, process: AbstractSubprocess, channel: str, fd: int) -> None:
@@ -176,16 +175,16 @@ class OutputDispatcher(Dispatcher):
176
175
  # )
177
176
 
178
177
  def remove_logs(self):
179
- for log in (self._normal_log, self._capture_log):
180
- if log is not None:
181
- for handler in log.handlers:
178
+ for l in (self._normal_log, self._capture_log):
179
+ if l is not None:
180
+ for handler in l.handlers:
182
181
  handler.remove() # type: ignore
183
182
  handler.reopen() # type: ignore
184
183
 
185
184
  def reopen_logs(self):
186
- for log in (self._normal_log, self._capture_log):
187
- if log is not None:
188
- for handler in log.handlers:
185
+ for l in (self._normal_log, self._capture_log):
186
+ if l is not None:
187
+ for handler in l.handlers:
189
188
  handler.reopen() # type: ignore
190
189
 
191
190
  def _log(self, data):
@@ -0,0 +1,68 @@
1
+ #!/usr/bin/env python3
2
+ # ruff: noqa: UP006 UP007
3
+ # @omlish-amalg ../scripts/supervisor.py
4
+ import itertools
5
+ import json
6
+ import typing as ta
7
+
8
+ from omlish.lite.journald import journald_log_handler_factory
9
+ from omlish.lite.logs import configure_standard_logging
10
+ from omlish.lite.marshal import unmarshal_obj
11
+
12
+ from .compat import ExitNow
13
+ from .configs import ServerConfig
14
+ from .context import ServerContext
15
+ from .states import SupervisorStates
16
+ from .supervisor import Supervisor
17
+
18
+
19
+ def main(
20
+ argv: ta.Optional[ta.Sequence[str]] = None,
21
+ *,
22
+ no_logging: bool = False,
23
+ ) -> None:
24
+ import argparse
25
+
26
+ parser = argparse.ArgumentParser()
27
+ parser.add_argument('config_file', metavar='config-file')
28
+ parser.add_argument('--no-journald', action='store_true')
29
+ args = parser.parse_args(argv)
30
+
31
+ #
32
+
33
+ if not (cf := args.config_file):
34
+ raise RuntimeError('No config file specified')
35
+
36
+ if not no_logging:
37
+ configure_standard_logging(
38
+ 'INFO',
39
+ handler_factory=journald_log_handler_factory if not args.no_journald else None,
40
+ )
41
+
42
+ #
43
+
44
+ # if we hup, restart by making a new Supervisor()
45
+ for epoch in itertools.count():
46
+ with open(cf) as f:
47
+ config_src = f.read()
48
+
49
+ config_dct = json.loads(config_src)
50
+ config: ServerConfig = unmarshal_obj(config_dct, ServerConfig)
51
+
52
+ context = ServerContext(
53
+ config,
54
+ epoch=epoch,
55
+ )
56
+
57
+ supervisor = Supervisor(context)
58
+ try:
59
+ supervisor.main()
60
+ except ExitNow:
61
+ pass
62
+
63
+ if context.state < SupervisorStates.RESTARTING:
64
+ break
65
+
66
+
67
+ if __name__ == '__main__':
68
+ main()
@@ -1,13 +1,11 @@
1
1
  # ruff: noqa: UP006 UP007
2
2
  import abc
3
3
  import errno
4
- import logging
5
4
  import select
6
5
  import sys
7
6
  import typing as ta
8
7
 
9
-
10
- log = logging.getLogger(__name__)
8
+ from omlish.lite.logs import log
11
9
 
12
10
 
13
11
  class BasePoller(abc.ABC):
@@ -1,7 +1,6 @@
1
1
  # ruff: noqa: UP006 UP007
2
2
  import errno
3
3
  import functools
4
- import logging
5
4
  import os
6
5
  import shlex
7
6
  import signal
@@ -9,6 +8,8 @@ import time
9
8
  import traceback
10
9
  import typing as ta
11
10
 
11
+ from omlish.lite.logs import log
12
+
12
13
  from .compat import as_bytes
13
14
  from .compat import as_string
14
15
  from .compat import close_fd
@@ -54,9 +55,6 @@ from .types import AbstractServerContext
54
55
  from .types import AbstractSubprocess
55
56
 
56
57
 
57
- log = logging.getLogger(__name__)
58
-
59
-
60
58
  @functools.total_ordering
61
59
  class Subprocess(AbstractSubprocess):
62
60
  """A class to manage a subprocess."""
@@ -1,22 +1,17 @@
1
- #!/usr/bin/env python3
2
1
  # ruff: noqa: UP006 UP007
3
- # @omlish-amalg ../scripts/supervisor.py
4
- import json
5
- import logging
6
2
  import signal
7
3
  import time
8
4
  import typing as ta
9
5
 
6
+ from omlish.lite.cached import cached_nullary
10
7
  from omlish.lite.check import check_not_none
11
- from omlish.lite.logs import configure_standard_logging
12
- from omlish.lite.marshal import unmarshal_obj
8
+ from omlish.lite.logs import log
13
9
 
14
10
  from .compat import ExitNow
15
11
  from .compat import as_string
16
12
  from .compat import decode_wait_status
17
13
  from .compat import signame
18
14
  from .configs import ProcessGroupConfig
19
- from .configs import ServerConfig
20
15
  from .context import ServerContext
21
16
  from .dispatchers import Dispatcher
22
17
  from .events import TICK_EVENTS
@@ -33,7 +28,8 @@ from .states import SupervisorStates
33
28
  from .states import get_process_state_description
34
29
 
35
30
 
36
- log = logging.getLogger(__name__)
31
+ def timeslice(period, when):
32
+ return int(when - (when % period))
37
33
 
38
34
 
39
35
  class Supervisor:
@@ -56,6 +52,11 @@ class Supervisor:
56
52
  return self._context.state
57
53
 
58
54
  def main(self) -> None:
55
+ self.setup()
56
+ self.run()
57
+
58
+ @cached_nullary
59
+ def setup(self) -> None:
59
60
  if not self._context.first:
60
61
  # prevent crash on libdispatch-based systems, at least for the first request
61
62
  self._context.cleanup_fds()
@@ -70,9 +71,11 @@ class Supervisor:
70
71
  # clean up old automatic logs
71
72
  self._context.clear_auto_child_logdir()
72
73
 
73
- self.run()
74
-
75
- def run(self) -> None:
74
+ def run(
75
+ self,
76
+ *,
77
+ callback: ta.Optional[ta.Callable[['Supervisor'], bool]] = None,
78
+ ) -> None:
76
79
  self._process_groups = {} # clear
77
80
  self._stop_groups = None # clear
78
81
 
@@ -90,12 +93,23 @@ class Supervisor:
90
93
  # writing pid file needs to come *after* daemonizing or pid will be wrong
91
94
  self._context.write_pidfile()
92
95
 
93
- self.runforever()
96
+ notify_event(SupervisorRunningEvent())
97
+
98
+ while True:
99
+ if callback is not None and not callback(self):
100
+ break
101
+
102
+ self._run_once()
94
103
 
95
104
  finally:
96
105
  self._context.cleanup()
97
106
 
98
- def diff_to_active(self):
107
+ class DiffToActive(ta.NamedTuple):
108
+ added: ta.List[ProcessGroupConfig]
109
+ changed: ta.List[ProcessGroupConfig]
110
+ removed: ta.List[ProcessGroupConfig]
111
+
112
+ def diff_to_active(self) -> DiffToActive:
99
113
  new = self._context.config.groups or []
100
114
  cur = [group.config for group in self._process_groups.values()]
101
115
 
@@ -107,7 +121,7 @@ class Supervisor:
107
121
 
108
122
  changed = [cand for cand in new if cand != curdict.get(cand.name, cand)]
109
123
 
110
- return added, changed, removed
124
+ return Supervisor.DiffToActive(added, changed, removed)
111
125
 
112
126
  def add_process_group(self, config: ProcessGroupConfig) -> bool:
113
127
  name = config.name
@@ -173,90 +187,84 @@ class Supervisor:
173
187
  # down, so push it back on to the end of the stop group queue
174
188
  self._stop_groups.append(group)
175
189
 
176
- def runforever(self) -> None:
177
- notify_event(SupervisorRunningEvent())
178
- timeout = 1 # this cannot be fewer than the smallest TickEvent (5)
190
+ def _run_once(self) -> None:
191
+ combined_map = {}
192
+ combined_map.update(self.get_process_map())
193
+
194
+ pgroups = list(self._process_groups.values())
195
+ pgroups.sort()
196
+
197
+ if self._context.state < SupervisorStates.RUNNING:
198
+ if not self._stopping:
199
+ # first time, set the stopping flag, do a notification and set stop_groups
200
+ self._stopping = True
201
+ self._stop_groups = pgroups[:]
202
+ notify_event(SupervisorStoppingEvent())
203
+
204
+ self._ordered_stop_groups_phase_1()
205
+
206
+ if not self.shutdown_report():
207
+ # if there are no unstopped processes (we're done killing everything), it's OK to shutdown or reload
208
+ raise ExitNow
179
209
 
180
- while True:
181
- combined_map = {}
182
- combined_map.update(self.get_process_map())
183
-
184
- pgroups = list(self._process_groups.values())
185
- pgroups.sort()
186
-
187
- if self._context.state < SupervisorStates.RUNNING:
188
- if not self._stopping:
189
- # first time, set the stopping flag, do a notification and set stop_groups
190
- self._stopping = True
191
- self._stop_groups = pgroups[:]
192
- notify_event(SupervisorStoppingEvent())
193
-
194
- self._ordered_stop_groups_phase_1()
195
-
196
- if not self.shutdown_report():
197
- # if there are no unstopped processes (we're done killing everything), it's OK to shutdown or reload
198
- raise ExitNow
199
-
200
- for fd, dispatcher in combined_map.items():
201
- if dispatcher.readable():
202
- self._context.poller.register_readable(fd)
203
- if dispatcher.writable():
204
- self._context.poller.register_writable(fd)
205
-
206
- r, w = self._context.poller.poll(timeout)
207
-
208
- for fd in r:
209
- if fd in combined_map:
210
- try:
211
- dispatcher = combined_map[fd]
212
- log.debug('read event caused by %r', dispatcher)
213
- dispatcher.handle_read_event()
214
- if not dispatcher.readable():
215
- self._context.poller.unregister_readable(fd)
216
- except ExitNow:
217
- raise
218
- except Exception: # noqa
219
- combined_map[fd].handle_error()
220
- else:
221
- # if the fd is not in combined_map, we should unregister it. otherwise, it will be polled every
222
- # time, which may cause 100% cpu usage
223
- log.debug('unexpected read event from fd %r', fd)
224
- try:
210
+ for fd, dispatcher in combined_map.items():
211
+ if dispatcher.readable():
212
+ self._context.poller.register_readable(fd)
213
+ if dispatcher.writable():
214
+ self._context.poller.register_writable(fd)
215
+
216
+ timeout = 1 # this cannot be fewer than the smallest TickEvent (5)
217
+ r, w = self._context.poller.poll(timeout)
218
+
219
+ for fd in r:
220
+ if fd in combined_map:
221
+ try:
222
+ dispatcher = combined_map[fd]
223
+ log.debug('read event caused by %r', dispatcher)
224
+ dispatcher.handle_read_event()
225
+ if not dispatcher.readable():
225
226
  self._context.poller.unregister_readable(fd)
226
- except Exception: # noqa
227
- pass
228
-
229
- for fd in w:
230
- if fd in combined_map:
231
- try:
232
- dispatcher = combined_map[fd]
233
- log.debug('write event caused by %r', dispatcher)
234
- dispatcher.handle_write_event()
235
- if not dispatcher.writable():
236
- self._context.poller.unregister_writable(fd)
237
- except ExitNow:
238
- raise
239
- except Exception: # noqa
240
- combined_map[fd].handle_error()
241
- else:
242
- log.debug('unexpected write event from fd %r', fd)
243
- try:
227
+ except ExitNow:
228
+ raise
229
+ except Exception: # noqa
230
+ combined_map[fd].handle_error()
231
+ else:
232
+ # if the fd is not in combined_map, we should unregister it. otherwise, it will be polled every
233
+ # time, which may cause 100% cpu usage
234
+ log.debug('unexpected read event from fd %r', fd)
235
+ try:
236
+ self._context.poller.unregister_readable(fd)
237
+ except Exception: # noqa
238
+ pass
239
+
240
+ for fd in w:
241
+ if fd in combined_map:
242
+ try:
243
+ dispatcher = combined_map[fd]
244
+ log.debug('write event caused by %r', dispatcher)
245
+ dispatcher.handle_write_event()
246
+ if not dispatcher.writable():
244
247
  self._context.poller.unregister_writable(fd)
245
- except Exception: # noqa
246
- pass
247
-
248
- for group in pgroups:
249
- group.transition()
248
+ except ExitNow:
249
+ raise
250
+ except Exception: # noqa
251
+ combined_map[fd].handle_error()
252
+ else:
253
+ log.debug('unexpected write event from fd %r', fd)
254
+ try:
255
+ self._context.poller.unregister_writable(fd)
256
+ except Exception: # noqa
257
+ pass
250
258
 
251
- self._reap()
252
- self._handle_signal()
253
- self._tick()
259
+ for group in pgroups:
260
+ group.transition()
254
261
 
255
- if self._context.state < SupervisorStates.RUNNING:
256
- self._ordered_stop_groups_phase_2()
262
+ self._reap()
263
+ self._handle_signal()
264
+ self._tick()
257
265
 
258
- if self._context.test:
259
- break
266
+ if self._context.state < SupervisorStates.RUNNING:
267
+ self._ordered_stop_groups_phase_2()
260
268
 
261
269
  def _tick(self, now: ta.Optional[float] = None) -> None:
262
270
  """Send one or more 'tick' events when the timeslice related to the period for the event type rolls over"""
@@ -325,52 +333,3 @@ class Supervisor:
325
333
 
326
334
  else:
327
335
  log.debug('received %s indicating nothing', signame(sig))
328
-
329
-
330
- def timeslice(period, when):
331
- return int(when - (when % period))
332
-
333
-
334
- def main(args=None, test=False):
335
- import argparse
336
-
337
- parser = argparse.ArgumentParser()
338
- parser.add_argument('config_file', metavar='config-file')
339
- args = parser.parse_args()
340
-
341
- configure_standard_logging('INFO')
342
-
343
- if not (cf := args.config_file):
344
- raise RuntimeError('No config file specified')
345
-
346
- # if we hup, restart by making a new Supervisor()
347
- first = True
348
- while True:
349
- with open(cf) as f:
350
- config_src = f.read()
351
- config_dct = json.loads(config_src)
352
- config: ServerConfig = unmarshal_obj(config_dct, ServerConfig)
353
-
354
- context = ServerContext(
355
- config,
356
- )
357
-
358
- context.first = first
359
- context.test = test
360
- go(context)
361
- # options.close_logger()
362
- first = False
363
- if test or (context.state < SupervisorStates.RESTARTING):
364
- break
365
-
366
-
367
- def go(context): # pragma: no cover
368
- d = Supervisor(context)
369
- try:
370
- d.main()
371
- except ExitNow:
372
- pass
373
-
374
-
375
- if __name__ == '__main__':
376
- main()
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: ominfra
3
- Version: 0.0.0.dev117
3
+ Version: 0.0.0.dev119
4
4
  Summary: ominfra
5
5
  Author: wrmsr
6
6
  License: BSD-3-Clause
@@ -12,8 +12,8 @@ Classifier: Operating System :: OS Independent
12
12
  Classifier: Operating System :: POSIX
13
13
  Requires-Python: >=3.12
14
14
  License-File: LICENSE
15
- Requires-Dist: omdev ==0.0.0.dev117
16
- Requires-Dist: omlish ==0.0.0.dev117
15
+ Requires-Dist: omdev ==0.0.0.dev119
16
+ Requires-Dist: omlish ==0.0.0.dev119
17
17
  Provides-Extra: all
18
18
  Requires-Dist: paramiko ~=3.5 ; extra == 'all'
19
19
  Requires-Dist: asyncssh ~=2.18 ; extra == 'all'