ominfra 0.0.0.dev126__py3-none-any.whl → 0.0.0.dev127__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. ominfra/clouds/aws/auth.py +1 -1
  2. ominfra/deploy/_executor.py +1 -1
  3. ominfra/deploy/poly/_main.py +1 -1
  4. ominfra/pyremote/_runcommands.py +1 -1
  5. ominfra/scripts/journald2aws.py +2 -2
  6. ominfra/scripts/supervisor.py +1796 -1218
  7. ominfra/supervisor/collections.py +52 -0
  8. ominfra/supervisor/context.py +2 -336
  9. ominfra/supervisor/datatypes.py +1 -63
  10. ominfra/supervisor/dispatchers.py +20 -324
  11. ominfra/supervisor/dispatchersimpl.py +342 -0
  12. ominfra/supervisor/groups.py +33 -111
  13. ominfra/supervisor/groupsimpl.py +86 -0
  14. ominfra/supervisor/inject.py +44 -19
  15. ominfra/supervisor/main.py +1 -1
  16. ominfra/supervisor/pipes.py +83 -0
  17. ominfra/supervisor/poller.py +6 -3
  18. ominfra/supervisor/privileges.py +65 -0
  19. ominfra/supervisor/processes.py +18 -0
  20. ominfra/supervisor/{process.py → processesimpl.py} +96 -330
  21. ominfra/supervisor/setup.py +38 -0
  22. ominfra/supervisor/setupimpl.py +261 -0
  23. ominfra/supervisor/signals.py +24 -16
  24. ominfra/supervisor/spawning.py +31 -0
  25. ominfra/supervisor/spawningimpl.py +347 -0
  26. ominfra/supervisor/supervisor.py +52 -77
  27. ominfra/supervisor/types.py +101 -45
  28. ominfra/supervisor/users.py +64 -0
  29. {ominfra-0.0.0.dev126.dist-info → ominfra-0.0.0.dev127.dist-info}/METADATA +3 -3
  30. {ominfra-0.0.0.dev126.dist-info → ominfra-0.0.0.dev127.dist-info}/RECORD +34 -23
  31. {ominfra-0.0.0.dev126.dist-info → ominfra-0.0.0.dev127.dist-info}/LICENSE +0 -0
  32. {ominfra-0.0.0.dev126.dist-info → ominfra-0.0.0.dev127.dist-info}/WHEEL +0 -0
  33. {ominfra-0.0.0.dev126.dist-info → ominfra-0.0.0.dev127.dist-info}/entry_points.txt +0 -0
  34. {ominfra-0.0.0.dev126.dist-info → ominfra-0.0.0.dev127.dist-info}/top_level.txt +0 -0
@@ -1,114 +1,16 @@
1
1
  # ruff: noqa: UP006 UP007
2
2
  import typing as ta
3
3
 
4
- from omlish.lite.check import check_isinstance
5
- from omlish.lite.typing import Func
6
-
4
+ from .collections import KeyedCollectionAccessors
7
5
  from .configs import ProcessGroupConfig
8
6
  from .events import EventCallbacks
9
7
  from .events import ProcessGroupAddedEvent
10
8
  from .events import ProcessGroupRemovedEvent
11
- from .states import ProcessState
12
- from .types import Dispatcher
13
9
  from .types import Process
14
10
  from .types import ProcessGroup
15
- from .types import ServerContext
16
-
17
-
18
- ##
19
11
 
20
12
 
21
- ProcessFactory = ta.NewType('ProcessFactory', Func[Process]) # (config: ProcessConfig, group: ProcessGroup)
22
-
23
-
24
- class ProcessGroupImpl(ProcessGroup):
25
- def __init__(
26
- self,
27
- config: ProcessGroupConfig,
28
- context: ServerContext,
29
- *,
30
- process_factory: ProcessFactory,
31
- ):
32
- super().__init__()
33
-
34
- self._config = config
35
- self._context = context
36
- self._process_factory = process_factory
37
-
38
- self._processes = {}
39
- for pconfig in self._config.processes or []:
40
- process = check_isinstance(self._process_factory(pconfig, self), Process)
41
- self._processes[pconfig.name] = process
42
-
43
- @property
44
- def config(self) -> ProcessGroupConfig:
45
- return self._config
46
-
47
- @property
48
- def name(self) -> str:
49
- return self._config.name
50
-
51
- @property
52
- def context(self) -> ServerContext:
53
- return self._context
54
-
55
- def __repr__(self):
56
- # repr can't return anything other than a native string, but the name might be unicode - a problem on Python 2.
57
- name = self._config.name
58
- return f'<{self.__class__.__name__} instance at {id(self)} named {name}>'
59
-
60
- def remove_logs(self) -> None:
61
- for process in self._processes.values():
62
- process.remove_logs()
63
-
64
- def reopen_logs(self) -> None:
65
- for process in self._processes.values():
66
- process.reopen_logs()
67
-
68
- def stop_all(self) -> None:
69
- processes = list(self._processes.values())
70
- processes.sort()
71
- processes.reverse() # stop in desc priority order
72
-
73
- for proc in processes:
74
- state = proc.get_state()
75
- if state == ProcessState.RUNNING:
76
- # RUNNING -> STOPPING
77
- proc.stop()
78
-
79
- elif state == ProcessState.STARTING:
80
- # STARTING -> STOPPING
81
- proc.stop()
82
-
83
- elif state == ProcessState.BACKOFF:
84
- # BACKOFF -> FATAL
85
- proc.give_up()
86
-
87
- def get_unstopped_processes(self) -> ta.List[Process]:
88
- return [x for x in self._processes.values() if not x.get_state().stopped]
89
-
90
- def get_dispatchers(self) -> ta.Dict[int, Dispatcher]:
91
- dispatchers: dict = {}
92
- for process in self._processes.values():
93
- dispatchers.update(process.get_dispatchers())
94
- return dispatchers
95
-
96
- def before_remove(self) -> None:
97
- pass
98
-
99
- def transition(self) -> None:
100
- for proc in self._processes.values():
101
- proc.transition()
102
-
103
- def after_setuid(self) -> None:
104
- for proc in self._processes.values():
105
- proc.create_auto_child_logs()
106
-
107
-
108
- ##
109
-
110
-
111
- class ProcessGroups:
13
+ class ProcessGroupManager(KeyedCollectionAccessors[str, ProcessGroup]):
112
14
  def __init__(
113
15
  self,
114
16
  *,
@@ -120,20 +22,17 @@ class ProcessGroups:
120
22
 
121
23
  self._by_name: ta.Dict[str, ProcessGroup] = {}
122
24
 
123
- def get(self, name: str) -> ta.Optional[ProcessGroup]:
124
- return self._by_name.get(name)
25
+ @property
26
+ def _by_key(self) -> ta.Mapping[str, ProcessGroup]:
27
+ return self._by_name
125
28
 
126
- def __getitem__(self, name: str) -> ProcessGroup:
127
- return self._by_name[name]
29
+ #
128
30
 
129
- def __len__(self) -> int:
130
- return len(self._by_name)
31
+ def all_processes(self) -> ta.Iterator[Process]:
32
+ for g in self:
33
+ yield from g
131
34
 
132
- def __iter__(self) -> ta.Iterator[ProcessGroup]:
133
- return iter(self._by_name.values())
134
-
135
- def all(self) -> ta.Mapping[str, ProcessGroup]:
136
- return self._by_name
35
+ #
137
36
 
138
37
  def add(self, group: ProcessGroup) -> None:
139
38
  if (name := group.name) in self._by_name:
@@ -155,3 +54,26 @@ class ProcessGroups:
155
54
  def clear(self) -> None:
156
55
  # FIXME: events?
157
56
  self._by_name.clear()
57
+
58
+ #
59
+
60
+ class Diff(ta.NamedTuple):
61
+ added: ta.List[ProcessGroupConfig]
62
+ changed: ta.List[ProcessGroupConfig]
63
+ removed: ta.List[ProcessGroupConfig]
64
+
65
+ def diff(self, new: ta.Sequence[ProcessGroupConfig]) -> Diff:
66
+ cur = [group.config for group in self]
67
+
68
+ cur_by_name = {cfg.name: cfg for cfg in cur}
69
+ new_by_name = {cfg.name: cfg for cfg in new}
70
+
71
+ added = [cand for cand in new if cand.name not in cur_by_name]
72
+ removed = [cand for cand in cur if cand.name not in new_by_name]
73
+ changed = [cand for cand in new if cand != cur_by_name.get(cand.name, cand)]
74
+
75
+ return ProcessGroupManager.Diff(
76
+ added,
77
+ changed,
78
+ removed,
79
+ )
@@ -0,0 +1,86 @@
1
+ # ruff: noqa: UP006 UP007
2
+ import typing as ta
3
+
4
+ from omlish.lite.check import check_isinstance
5
+ from omlish.lite.typing import Func2
6
+
7
+ from .configs import ProcessConfig
8
+ from .configs import ProcessGroupConfig
9
+ from .states import ProcessState
10
+ from .types import Process
11
+ from .types import ProcessGroup
12
+
13
+
14
+ class ProcessFactory(Func2[ProcessConfig, ProcessGroup, Process]):
15
+ pass
16
+
17
+
18
+ class ProcessGroupImpl(ProcessGroup):
19
+ def __init__(
20
+ self,
21
+ config: ProcessGroupConfig,
22
+ *,
23
+ process_factory: ProcessFactory,
24
+ ):
25
+ super().__init__()
26
+
27
+ self._config = config
28
+ self._process_factory = process_factory
29
+
30
+ by_name: ta.Dict[str, Process] = {}
31
+ for pconfig in self._config.processes or []:
32
+ p = check_isinstance(self._process_factory(pconfig, self), Process)
33
+ if p.name in by_name:
34
+ raise KeyError(f'name {p.name} of process {p} already registered by {by_name[p.name]}')
35
+ by_name[pconfig.name] = p
36
+ self._by_name = by_name
37
+
38
+ @property
39
+ def _by_key(self) -> ta.Mapping[str, Process]:
40
+ return self._by_name
41
+
42
+ #
43
+
44
+ def __repr__(self) -> str:
45
+ return f'<{self.__class__.__name__} instance at {id(self)} named {self._config.name}>'
46
+
47
+ #
48
+
49
+ @property
50
+ def name(self) -> str:
51
+ return self._config.name
52
+
53
+ @property
54
+ def config(self) -> ProcessGroupConfig:
55
+ return self._config
56
+
57
+ @property
58
+ def by_name(self) -> ta.Mapping[str, Process]:
59
+ return self._by_name
60
+
61
+ #
62
+
63
+ def get_unstopped_processes(self) -> ta.List[Process]:
64
+ return [x for x in self if not x.get_state().stopped]
65
+
66
+ def stop_all(self) -> None:
67
+ processes = list(self._by_name.values())
68
+ processes.sort()
69
+ processes.reverse() # stop in desc priority order
70
+
71
+ for proc in processes:
72
+ state = proc.get_state()
73
+ if state == ProcessState.RUNNING:
74
+ # RUNNING -> STOPPING
75
+ proc.stop()
76
+
77
+ elif state == ProcessState.STARTING:
78
+ # STARTING -> STOPPING
79
+ proc.stop()
80
+
81
+ elif state == ProcessState.BACKOFF:
82
+ # BACKOFF -> FATAL
83
+ proc.give_up()
84
+
85
+ def before_remove(self) -> None:
86
+ pass
@@ -7,27 +7,33 @@ from omlish.lite.inject import inj
7
7
 
8
8
  from .configs import ServerConfig
9
9
  from .context import ServerContextImpl
10
- from .context import ServerEpoch
11
- from .dispatchers import InputDispatcherImpl
12
- from .dispatchers import OutputDispatcherImpl
10
+ from .dispatchersimpl import InputDispatcherImpl
11
+ from .dispatchersimpl import OutputDispatcherImpl
13
12
  from .events import EventCallbacks
14
- from .groups import ProcessFactory
15
- from .groups import ProcessGroupImpl
13
+ from .groups import ProcessGroupManager
14
+ from .groupsimpl import ProcessFactory
15
+ from .groupsimpl import ProcessGroupImpl
16
16
  from .poller import Poller
17
17
  from .poller import get_poller_impl
18
- from .process import InheritedFds
19
- from .process import InputDispatcherFactory
20
- from .process import OutputDispatcherFactory
21
- from .process import ProcessImpl
18
+ from .processes import PidHistory
19
+ from .processesimpl import ProcessImpl
20
+ from .processesimpl import ProcessSpawningFactory
21
+ from .setup import DaemonizeListener
22
+ from .setup import DaemonizeListeners
23
+ from .setup import SupervisorUser
24
+ from .setupimpl import SupervisorSetup
25
+ from .setupimpl import SupervisorSetupImpl
22
26
  from .signals import SignalReceiver
27
+ from .spawningimpl import InheritedFds
28
+ from .spawningimpl import InputDispatcherFactory
29
+ from .spawningimpl import OutputDispatcherFactory
30
+ from .spawningimpl import ProcessSpawningImpl
23
31
  from .supervisor import ProcessGroupFactory
24
- from .supervisor import ProcessGroups
25
32
  from .supervisor import SignalHandler
26
33
  from .supervisor import Supervisor
27
34
  from .types import ServerContext
28
-
29
-
30
- ##
35
+ from .types import ServerEpoch
36
+ from .users import get_user
31
37
 
32
38
 
33
39
  def bind_server(
@@ -39,7 +45,12 @@ def bind_server(
39
45
  lst: ta.List[InjectorBindingOrBindings] = [
40
46
  inj.bind(config),
41
47
 
42
- inj.bind(get_poller_impl(), key=Poller, singleton=True),
48
+ inj.bind_array_type(DaemonizeListener, DaemonizeListeners),
49
+
50
+ inj.bind(SupervisorSetupImpl, singleton=True),
51
+ inj.bind(SupervisorSetup, to_key=SupervisorSetupImpl),
52
+
53
+ inj.bind(DaemonizeListener, array=True, to_key=Poller),
43
54
 
44
55
  inj.bind(ServerContextImpl, singleton=True),
45
56
  inj.bind(ServerContext, to_key=ServerContextImpl),
@@ -49,14 +60,18 @@ def bind_server(
49
60
  inj.bind(SignalReceiver, singleton=True),
50
61
 
51
62
  inj.bind(SignalHandler, singleton=True),
52
- inj.bind(ProcessGroups, singleton=True),
63
+ inj.bind(ProcessGroupManager, singleton=True),
53
64
  inj.bind(Supervisor, singleton=True),
54
65
 
55
- inj.bind_factory(ProcessGroupFactory, ProcessGroupImpl),
56
- inj.bind_factory(ProcessFactory, ProcessImpl),
66
+ inj.bind(PidHistory()),
67
+
68
+ inj.bind_factory(ProcessGroupImpl, ProcessGroupFactory),
69
+ inj.bind_factory(ProcessImpl, ProcessFactory),
70
+
71
+ inj.bind_factory(ProcessSpawningImpl, ProcessSpawningFactory),
57
72
 
58
- inj.bind_factory(OutputDispatcherFactory, OutputDispatcherImpl),
59
- inj.bind_factory(InputDispatcherFactory, InputDispatcherImpl),
73
+ inj.bind_factory(OutputDispatcherImpl, OutputDispatcherFactory),
74
+ inj.bind_factory(InputDispatcherImpl, InputDispatcherFactory),
60
75
  ]
61
76
 
62
77
  #
@@ -68,4 +83,14 @@ def bind_server(
68
83
 
69
84
  #
70
85
 
86
+ if config.user is not None:
87
+ user = get_user(config.user)
88
+ lst.append(inj.bind(user, key=SupervisorUser))
89
+
90
+ #
91
+
92
+ lst.append(inj.bind(get_poller_impl(), key=Poller, singleton=True))
93
+
94
+ #
95
+
71
96
  return inj.as_bindings(*lst)
@@ -44,7 +44,7 @@ from .configs import prepare_server_config
44
44
  from .context import ServerContextImpl
45
45
  from .context import ServerEpoch
46
46
  from .inject import bind_server
47
- from .process import InheritedFds
47
+ from .spawningimpl import InheritedFds
48
48
  from .states import SupervisorState
49
49
  from .supervisor import Supervisor
50
50
  from .utils import ExitNow
@@ -0,0 +1,83 @@
1
+ # ruff: noqa: UP006 UP007
2
+ import dataclasses as dc
3
+ import fcntl
4
+ import os
5
+ import typing as ta
6
+
7
+ from .utils import close_fd
8
+
9
+
10
+ @dc.dataclass(frozen=True)
11
+ class ProcessPipes:
12
+ child_stdin: ta.Optional[int] = None
13
+ stdin: ta.Optional[int] = None
14
+
15
+ stdout: ta.Optional[int] = None
16
+ child_stdout: ta.Optional[int] = None
17
+
18
+ stderr: ta.Optional[int] = None
19
+ child_stderr: ta.Optional[int] = None
20
+
21
+ def child_fds(self) -> ta.List[int]:
22
+ return [fd for fd in [self.child_stdin, self.child_stdout, self.child_stderr] if fd is not None]
23
+
24
+ def parent_fds(self) -> ta.List[int]:
25
+ return [fd for fd in [self.stdin, self.stdout, self.stderr] if fd is not None]
26
+
27
+
28
+ def make_process_pipes(stderr=True) -> ProcessPipes:
29
+ """
30
+ Create pipes for parent to child stdin/stdout/stderr communications. Open fd in non-blocking mode so we can
31
+ read them in the mainloop without blocking. If stderr is False, don't create a pipe for stderr.
32
+ """
33
+
34
+ pipes: ta.Dict[str, ta.Optional[int]] = {
35
+ 'child_stdin': None,
36
+ 'stdin': None,
37
+
38
+ 'stdout': None,
39
+ 'child_stdout': None,
40
+
41
+ 'stderr': None,
42
+ 'child_stderr': None,
43
+ }
44
+
45
+ try:
46
+ pipes['child_stdin'], pipes['stdin'] = os.pipe()
47
+ pipes['stdout'], pipes['child_stdout'] = os.pipe()
48
+
49
+ if stderr:
50
+ pipes['stderr'], pipes['child_stderr'] = os.pipe()
51
+
52
+ for fd in (
53
+ pipes['stdout'],
54
+ pipes['stderr'],
55
+ pipes['stdin'],
56
+ ):
57
+ if fd is not None:
58
+ flags = fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NDELAY
59
+ fcntl.fcntl(fd, fcntl.F_SETFL, flags)
60
+
61
+ return ProcessPipes(**pipes)
62
+
63
+ except OSError:
64
+ for fd in pipes.values():
65
+ if fd is not None:
66
+ close_fd(fd)
67
+
68
+ raise
69
+
70
+
71
+ def close_pipes(pipes: ProcessPipes) -> None:
72
+ close_parent_pipes(pipes)
73
+ close_child_pipes(pipes)
74
+
75
+
76
+ def close_parent_pipes(pipes: ProcessPipes) -> None:
77
+ for fd in pipes.parent_fds():
78
+ close_fd(fd)
79
+
80
+
81
+ def close_child_pipes(pipes: ProcessPipes) -> None:
82
+ for fd in pipes.child_fds():
83
+ close_fd(fd)
@@ -7,8 +7,10 @@ import typing as ta
7
7
 
8
8
  from omlish.lite.logs import log
9
9
 
10
+ from .setup import DaemonizeListener
10
11
 
11
- class Poller(abc.ABC):
12
+
13
+ class Poller(DaemonizeListener, abc.ABC):
12
14
  def __init__(self) -> None:
13
15
  super().__init__()
14
16
 
@@ -226,8 +228,9 @@ else:
226
228
 
227
229
  def get_poller_impl() -> ta.Type[Poller]:
228
230
  if (
229
- sys.platform == 'darwin' or sys.platform.startswith('freebsd') and
230
- hasattr(select, 'kqueue') and KqueuePoller is not None
231
+ (sys.platform == 'darwin' or sys.platform.startswith('freebsd')) and
232
+ hasattr(select, 'kqueue') and
233
+ KqueuePoller is not None
231
234
  ):
232
235
  return KqueuePoller
233
236
  elif hasattr(select, 'poll'):
@@ -0,0 +1,65 @@
1
+ # ruff: noqa: UP006 UP007
2
+ import grp
3
+ import os
4
+ import pwd
5
+ import typing as ta
6
+
7
+
8
+ def drop_privileges(user: ta.Union[int, str, None]) -> ta.Optional[str]:
9
+ """
10
+ Drop privileges to become the specified user, which may be a username or uid. Called for supervisord startup
11
+ and when spawning subprocesses. Returns None on success or a string error message if privileges could not be
12
+ dropped.
13
+ """
14
+
15
+ if user is None:
16
+ return 'No user specified to setuid to!'
17
+
18
+ # get uid for user, which can be a number or username
19
+ try:
20
+ uid = int(user)
21
+ except ValueError:
22
+ try:
23
+ pwrec = pwd.getpwnam(user) # type: ignore
24
+ except KeyError:
25
+ return f"Can't find username {user!r}"
26
+ uid = pwrec[2]
27
+ else:
28
+ try:
29
+ pwrec = pwd.getpwuid(uid)
30
+ except KeyError:
31
+ return f"Can't find uid {uid!r}"
32
+
33
+ current_uid = os.getuid()
34
+
35
+ if current_uid == uid:
36
+ # do nothing and return successfully if the uid is already the current one. this allows a supervisord
37
+ # running as an unprivileged user "foo" to start a process where the config has "user=foo" (same user) in
38
+ # it.
39
+ return None
40
+
41
+ if current_uid != 0:
42
+ return "Can't drop privilege as nonroot user"
43
+
44
+ gid = pwrec[3]
45
+ if hasattr(os, 'setgroups'):
46
+ user = pwrec[0]
47
+ groups = [grprec[2] for grprec in grp.getgrall() if user in grprec[3]]
48
+
49
+ # always put our primary gid first in this list, otherwise we can lose group info since sometimes the first
50
+ # group in the setgroups list gets overwritten on the subsequent setgid call (at least on freebsd 9 with
51
+ # python 2.7 - this will be safe though for all unix /python version combos)
52
+ groups.insert(0, gid)
53
+ try:
54
+ os.setgroups(groups)
55
+ except OSError:
56
+ return 'Could not set groups of effective user'
57
+
58
+ try:
59
+ os.setgid(gid)
60
+ except OSError:
61
+ return 'Could not set group id of effective user'
62
+
63
+ os.setuid(uid)
64
+
65
+ return None
@@ -0,0 +1,18 @@
1
+ # ruff: noqa: UP006 UP007
2
+ import typing as ta
3
+
4
+ from .types import Process
5
+
6
+
7
+ ##
8
+
9
+
10
+ class ProcessStateError(RuntimeError):
11
+ pass
12
+
13
+
14
+ ##
15
+
16
+
17
+ class PidHistory(ta.Dict[int, Process]):
18
+ pass