ominfra 0.0.0.dev118__py3-none-any.whl → 0.0.0.dev120__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ominfra/deploy/_executor.py +6 -2
- ominfra/deploy/poly/_main.py +6 -2
- ominfra/journald/fields.py +187 -0
- ominfra/journald/tailer.py +375 -312
- ominfra/pyremote/_runcommands.py +6 -2
- ominfra/scripts/journald2aws.py +381 -314
- ominfra/scripts/supervisor.py +697 -464
- ominfra/supervisor/__main__.py +1 -1
- ominfra/supervisor/context.py +50 -25
- ominfra/supervisor/dispatchers.py +12 -13
- ominfra/supervisor/events.py +4 -7
- ominfra/supervisor/main.py +68 -0
- ominfra/supervisor/poller.py +1 -3
- ominfra/supervisor/process.py +10 -11
- ominfra/supervisor/supervisor.py +161 -193
- {ominfra-0.0.0.dev118.dist-info → ominfra-0.0.0.dev120.dist-info}/METADATA +3 -3
- {ominfra-0.0.0.dev118.dist-info → ominfra-0.0.0.dev120.dist-info}/RECORD +21 -19
- {ominfra-0.0.0.dev118.dist-info → ominfra-0.0.0.dev120.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev118.dist-info → ominfra-0.0.0.dev120.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev118.dist-info → ominfra-0.0.0.dev120.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev118.dist-info → ominfra-0.0.0.dev120.dist-info}/top_level.txt +0 -0
ominfra/scripts/supervisor.py
CHANGED
@@ -2,12 +2,13 @@
|
|
2
2
|
# noinspection DuplicatedCode
|
3
3
|
# @omlish-lite
|
4
4
|
# @omlish-script
|
5
|
-
# @omlish-amalg-output ../supervisor/
|
6
|
-
# ruff: noqa: N802 UP006 UP007 UP036
|
5
|
+
# @omlish-amalg-output ../supervisor/main.py
|
6
|
+
# ruff: noqa: N802 UP006 UP007 UP012 UP036
|
7
7
|
import abc
|
8
8
|
import base64
|
9
9
|
import collections.abc
|
10
10
|
import contextlib
|
11
|
+
import ctypes as ct
|
11
12
|
import dataclasses as dc
|
12
13
|
import datetime
|
13
14
|
import decimal
|
@@ -17,9 +18,11 @@ import fcntl
|
|
17
18
|
import fractions
|
18
19
|
import functools
|
19
20
|
import grp
|
21
|
+
import itertools
|
20
22
|
import json
|
21
23
|
import logging
|
22
24
|
import os
|
25
|
+
import os.path
|
23
26
|
import pwd
|
24
27
|
import re
|
25
28
|
import resource
|
@@ -28,6 +31,7 @@ import shlex
|
|
28
31
|
import signal
|
29
32
|
import stat
|
30
33
|
import sys
|
34
|
+
import syslog
|
31
35
|
import tempfile
|
32
36
|
import threading
|
33
37
|
import time
|
@@ -466,240 +470,28 @@ class NoPermissionError(ProcessError):
|
|
466
470
|
|
467
471
|
|
468
472
|
########################################
|
469
|
-
#
|
470
|
-
|
471
|
-
|
472
|
-
log = logging.getLogger(__name__)
|
473
|
-
|
474
|
-
|
475
|
-
class BasePoller(abc.ABC):
|
476
|
-
|
477
|
-
def __init__(self) -> None:
|
478
|
-
super().__init__()
|
479
|
-
|
480
|
-
@abc.abstractmethod
|
481
|
-
def register_readable(self, fd: int) -> None:
|
482
|
-
raise NotImplementedError
|
483
|
-
|
484
|
-
@abc.abstractmethod
|
485
|
-
def register_writable(self, fd: int) -> None:
|
486
|
-
raise NotImplementedError
|
487
|
-
|
488
|
-
@abc.abstractmethod
|
489
|
-
def unregister_readable(self, fd: int) -> None:
|
490
|
-
raise NotImplementedError
|
491
|
-
|
492
|
-
@abc.abstractmethod
|
493
|
-
def unregister_writable(self, fd: int) -> None:
|
494
|
-
raise NotImplementedError
|
495
|
-
|
496
|
-
@abc.abstractmethod
|
497
|
-
def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[int], ta.List[int]]:
|
498
|
-
raise NotImplementedError
|
499
|
-
|
500
|
-
def before_daemonize(self) -> None: # noqa
|
501
|
-
pass
|
502
|
-
|
503
|
-
def after_daemonize(self) -> None: # noqa
|
504
|
-
pass
|
505
|
-
|
506
|
-
def close(self) -> None: # noqa
|
507
|
-
pass
|
508
|
-
|
509
|
-
|
510
|
-
class SelectPoller(BasePoller):
|
511
|
-
|
512
|
-
def __init__(self) -> None:
|
513
|
-
super().__init__()
|
514
|
-
|
515
|
-
self._readables: ta.Set[int] = set()
|
516
|
-
self._writables: ta.Set[int] = set()
|
517
|
-
|
518
|
-
def register_readable(self, fd: int) -> None:
|
519
|
-
self._readables.add(fd)
|
520
|
-
|
521
|
-
def register_writable(self, fd: int) -> None:
|
522
|
-
self._writables.add(fd)
|
523
|
-
|
524
|
-
def unregister_readable(self, fd: int) -> None:
|
525
|
-
self._readables.discard(fd)
|
526
|
-
|
527
|
-
def unregister_writable(self, fd: int) -> None:
|
528
|
-
self._writables.discard(fd)
|
529
|
-
|
530
|
-
def unregister_all(self) -> None:
|
531
|
-
self._readables.clear()
|
532
|
-
self._writables.clear()
|
473
|
+
# ../../../omlish/lite/cached.py
|
533
474
|
|
534
|
-
def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[int], ta.List[int]]:
|
535
|
-
try:
|
536
|
-
r, w, x = select.select(
|
537
|
-
self._readables,
|
538
|
-
self._writables,
|
539
|
-
[], timeout,
|
540
|
-
)
|
541
|
-
except OSError as err:
|
542
|
-
if err.args[0] == errno.EINTR:
|
543
|
-
log.debug('EINTR encountered in poll')
|
544
|
-
return [], []
|
545
|
-
if err.args[0] == errno.EBADF:
|
546
|
-
log.debug('EBADF encountered in poll')
|
547
|
-
self.unregister_all()
|
548
|
-
return [], []
|
549
|
-
raise
|
550
|
-
return r, w
|
551
|
-
|
552
|
-
|
553
|
-
class PollPoller(BasePoller):
|
554
|
-
_READ = select.POLLIN | select.POLLPRI | select.POLLHUP
|
555
|
-
_WRITE = select.POLLOUT
|
556
475
|
|
557
|
-
|
476
|
+
class _cached_nullary: # noqa
|
477
|
+
def __init__(self, fn):
|
558
478
|
super().__init__()
|
479
|
+
self._fn = fn
|
480
|
+
self._value = self._missing = object()
|
481
|
+
functools.update_wrapper(self, fn)
|
559
482
|
|
560
|
-
|
561
|
-
self.
|
562
|
-
|
563
|
-
|
564
|
-
def register_readable(self, fd: int) -> None:
|
565
|
-
self._poller.register(fd, self._READ)
|
566
|
-
self._readables.add(fd)
|
567
|
-
|
568
|
-
def register_writable(self, fd: int) -> None:
|
569
|
-
self._poller.register(fd, self._WRITE)
|
570
|
-
self._writables.add(fd)
|
483
|
+
def __call__(self, *args, **kwargs): # noqa
|
484
|
+
if self._value is self._missing:
|
485
|
+
self._value = self._fn()
|
486
|
+
return self._value
|
571
487
|
|
572
|
-
def
|
573
|
-
self.
|
574
|
-
|
575
|
-
if fd in self._writables:
|
576
|
-
self._poller.register(fd, self._WRITE)
|
488
|
+
def __get__(self, instance, owner): # noqa
|
489
|
+
bound = instance.__dict__[self._fn.__name__] = self.__class__(self._fn.__get__(instance, owner))
|
490
|
+
return bound
|
577
491
|
|
578
|
-
def unregister_writable(self, fd: int) -> None:
|
579
|
-
self._writables.discard(fd)
|
580
|
-
self._poller.unregister(fd)
|
581
|
-
if fd in self._readables:
|
582
|
-
self._poller.register(fd, self._READ)
|
583
492
|
|
584
|
-
|
585
|
-
|
586
|
-
readables, writables = [], []
|
587
|
-
for fd, eventmask in fds:
|
588
|
-
if self._ignore_invalid(fd, eventmask):
|
589
|
-
continue
|
590
|
-
if eventmask & self._READ:
|
591
|
-
readables.append(fd)
|
592
|
-
if eventmask & self._WRITE:
|
593
|
-
writables.append(fd)
|
594
|
-
return readables, writables
|
595
|
-
|
596
|
-
def _poll_fds(self, timeout: float) -> ta.List[ta.Tuple[int, int]]:
|
597
|
-
try:
|
598
|
-
return self._poller.poll(timeout * 1000)
|
599
|
-
except OSError as err:
|
600
|
-
if err.args[0] == errno.EINTR:
|
601
|
-
log.debug('EINTR encountered in poll')
|
602
|
-
return []
|
603
|
-
raise
|
604
|
-
|
605
|
-
def _ignore_invalid(self, fd: int, eventmask: int) -> bool:
|
606
|
-
if eventmask & select.POLLNVAL:
|
607
|
-
# POLLNVAL means `fd` value is invalid, not open. When a process quits it's `fd`s are closed so there is no
|
608
|
-
# more reason to keep this `fd` registered If the process restarts it's `fd`s are registered again.
|
609
|
-
self._poller.unregister(fd)
|
610
|
-
self._readables.discard(fd)
|
611
|
-
self._writables.discard(fd)
|
612
|
-
return True
|
613
|
-
return False
|
614
|
-
|
615
|
-
|
616
|
-
if sys.platform == 'darwin' or sys.platform.startswith('freebsd'):
|
617
|
-
class KqueuePoller(BasePoller):
|
618
|
-
max_events = 1000
|
619
|
-
|
620
|
-
def __init__(self) -> None:
|
621
|
-
super().__init__()
|
622
|
-
|
623
|
-
self._kqueue: ta.Optional[ta.Any] = select.kqueue()
|
624
|
-
self._readables: set[int] = set()
|
625
|
-
self._writables: set[int] = set()
|
626
|
-
|
627
|
-
def register_readable(self, fd: int) -> None:
|
628
|
-
self._readables.add(fd)
|
629
|
-
kevent = select.kevent(fd, filter=select.KQ_FILTER_READ, flags=select.KQ_EV_ADD)
|
630
|
-
self._kqueue_control(fd, kevent)
|
631
|
-
|
632
|
-
def register_writable(self, fd: int) -> None:
|
633
|
-
self._writables.add(fd)
|
634
|
-
kevent = select.kevent(fd, filter=select.KQ_FILTER_WRITE, flags=select.KQ_EV_ADD)
|
635
|
-
self._kqueue_control(fd, kevent)
|
636
|
-
|
637
|
-
def unregister_readable(self, fd: int) -> None:
|
638
|
-
kevent = select.kevent(fd, filter=select.KQ_FILTER_READ, flags=select.KQ_EV_DELETE)
|
639
|
-
self._readables.discard(fd)
|
640
|
-
self._kqueue_control(fd, kevent)
|
641
|
-
|
642
|
-
def unregister_writable(self, fd: int) -> None:
|
643
|
-
kevent = select.kevent(fd, filter=select.KQ_FILTER_WRITE, flags=select.KQ_EV_DELETE)
|
644
|
-
self._writables.discard(fd)
|
645
|
-
self._kqueue_control(fd, kevent)
|
646
|
-
|
647
|
-
def _kqueue_control(self, fd: int, kevent: 'select.kevent') -> None:
|
648
|
-
try:
|
649
|
-
self._kqueue.control([kevent], 0) # type: ignore
|
650
|
-
except OSError as error:
|
651
|
-
if error.errno == errno.EBADF:
|
652
|
-
log.debug('EBADF encountered in kqueue. Invalid file descriptor %s', fd)
|
653
|
-
else:
|
654
|
-
raise
|
655
|
-
|
656
|
-
def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[int], ta.List[int]]:
|
657
|
-
readables, writables = [], [] # type: ignore
|
658
|
-
|
659
|
-
try:
|
660
|
-
kevents = self._kqueue.control(None, self.max_events, timeout) # type: ignore
|
661
|
-
except OSError as error:
|
662
|
-
if error.errno == errno.EINTR:
|
663
|
-
log.debug('EINTR encountered in poll')
|
664
|
-
return readables, writables
|
665
|
-
raise
|
666
|
-
|
667
|
-
for kevent in kevents:
|
668
|
-
if kevent.filter == select.KQ_FILTER_READ:
|
669
|
-
readables.append(kevent.ident)
|
670
|
-
if kevent.filter == select.KQ_FILTER_WRITE:
|
671
|
-
writables.append(kevent.ident)
|
672
|
-
|
673
|
-
return readables, writables
|
674
|
-
|
675
|
-
def before_daemonize(self) -> None:
|
676
|
-
self.close()
|
677
|
-
|
678
|
-
def after_daemonize(self) -> None:
|
679
|
-
self._kqueue = select.kqueue()
|
680
|
-
for fd in self._readables:
|
681
|
-
self.register_readable(fd)
|
682
|
-
for fd in self._writables:
|
683
|
-
self.register_writable(fd)
|
684
|
-
|
685
|
-
def close(self) -> None:
|
686
|
-
self._kqueue.close() # type: ignore
|
687
|
-
self._kqueue = None
|
688
|
-
|
689
|
-
else:
|
690
|
-
KqueuePoller = None
|
691
|
-
|
692
|
-
|
693
|
-
Poller: ta.Type[BasePoller]
|
694
|
-
if (
|
695
|
-
sys.platform == 'darwin' or sys.platform.startswith('freebsd') and
|
696
|
-
hasattr(select, 'kqueue') and KqueuePoller is not None
|
697
|
-
):
|
698
|
-
Poller = KqueuePoller
|
699
|
-
elif hasattr(select, 'poll'):
|
700
|
-
Poller = PollPoller
|
701
|
-
else:
|
702
|
-
Poller = SelectPoller
|
493
|
+
def cached_nullary(fn: ta.Callable[..., T]) -> ta.Callable[..., T]:
|
494
|
+
return _cached_nullary(fn)
|
703
495
|
|
704
496
|
|
705
497
|
########################################
|
@@ -1012,6 +804,164 @@ def get_supervisor_state_description(code: SupervisorState) -> str:
|
|
1012
804
|
return check_not_none(_supervisor_states_by_code.get(code))
|
1013
805
|
|
1014
806
|
|
807
|
+
########################################
|
808
|
+
# ../../../omlish/lite/journald.py
|
809
|
+
|
810
|
+
|
811
|
+
##
|
812
|
+
|
813
|
+
|
814
|
+
class sd_iovec(ct.Structure): # noqa
|
815
|
+
pass
|
816
|
+
|
817
|
+
|
818
|
+
sd_iovec._fields_ = [
|
819
|
+
('iov_base', ct.c_void_p), # Pointer to data.
|
820
|
+
('iov_len', ct.c_size_t), # Length of data.
|
821
|
+
]
|
822
|
+
|
823
|
+
|
824
|
+
##
|
825
|
+
|
826
|
+
|
827
|
+
@cached_nullary
|
828
|
+
def sd_libsystemd() -> ta.Any:
|
829
|
+
lib = ct.CDLL('libsystemd.so.0')
|
830
|
+
|
831
|
+
lib.sd_journal_sendv = lib['sd_journal_sendv'] # type: ignore
|
832
|
+
lib.sd_journal_sendv.restype = ct.c_int
|
833
|
+
lib.sd_journal_sendv.argtypes = [ct.POINTER(sd_iovec), ct.c_int]
|
834
|
+
|
835
|
+
return lib
|
836
|
+
|
837
|
+
|
838
|
+
@cached_nullary
|
839
|
+
def sd_try_libsystemd() -> ta.Optional[ta.Any]:
|
840
|
+
try:
|
841
|
+
return sd_libsystemd()
|
842
|
+
except OSError: # noqa
|
843
|
+
return None
|
844
|
+
|
845
|
+
|
846
|
+
##
|
847
|
+
|
848
|
+
|
849
|
+
def sd_journald_send(**fields: str) -> int:
|
850
|
+
lib = sd_libsystemd()
|
851
|
+
|
852
|
+
msgs = [
|
853
|
+
f'{k.upper()}={v}\0'.encode('utf-8')
|
854
|
+
for k, v in fields.items()
|
855
|
+
]
|
856
|
+
|
857
|
+
vec = (sd_iovec * len(msgs))()
|
858
|
+
cl = (ct.c_char_p * len(msgs))() # noqa
|
859
|
+
for i in range(len(msgs)):
|
860
|
+
vec[i].iov_base = ct.cast(ct.c_char_p(msgs[i]), ct.c_void_p)
|
861
|
+
vec[i].iov_len = len(msgs[i]) - 1
|
862
|
+
|
863
|
+
return lib.sd_journal_sendv(vec, len(msgs))
|
864
|
+
|
865
|
+
|
866
|
+
##
|
867
|
+
|
868
|
+
|
869
|
+
SD_LOG_LEVEL_MAP: ta.Mapping[int, int] = {
|
870
|
+
logging.FATAL: syslog.LOG_EMERG, # system is unusable
|
871
|
+
# LOG_ALERT ? # action must be taken immediately
|
872
|
+
logging.CRITICAL: syslog.LOG_CRIT,
|
873
|
+
logging.ERROR: syslog.LOG_ERR,
|
874
|
+
logging.WARNING: syslog.LOG_WARNING,
|
875
|
+
# LOG_NOTICE ? # normal but significant condition
|
876
|
+
logging.INFO: syslog.LOG_INFO,
|
877
|
+
logging.DEBUG: syslog.LOG_DEBUG,
|
878
|
+
}
|
879
|
+
|
880
|
+
|
881
|
+
class JournaldLogHandler(logging.Handler):
|
882
|
+
"""
|
883
|
+
TODO:
|
884
|
+
- fallback handler for when this barfs
|
885
|
+
"""
|
886
|
+
|
887
|
+
def __init__(
|
888
|
+
self,
|
889
|
+
*,
|
890
|
+
use_formatter_output: bool = False,
|
891
|
+
) -> None:
|
892
|
+
super().__init__()
|
893
|
+
|
894
|
+
sd_libsystemd()
|
895
|
+
|
896
|
+
self._use_formatter_output = use_formatter_output
|
897
|
+
|
898
|
+
#
|
899
|
+
|
900
|
+
EXTRA_RECORD_ATTRS_BY_JOURNALD_FIELD: ta.ClassVar[ta.Mapping[str, str]] = {
|
901
|
+
'name': 'name',
|
902
|
+
'module': 'module',
|
903
|
+
'exception': 'exc_text',
|
904
|
+
'thread_name': 'threadName',
|
905
|
+
'task_name': 'taskName',
|
906
|
+
}
|
907
|
+
|
908
|
+
def make_fields(self, record: logging.LogRecord) -> ta.Mapping[str, str]:
|
909
|
+
formatter_message = self.format(record)
|
910
|
+
if self._use_formatter_output:
|
911
|
+
message = formatter_message
|
912
|
+
else:
|
913
|
+
message = record.message
|
914
|
+
|
915
|
+
fields: dict[str, str] = {
|
916
|
+
'message': message,
|
917
|
+
'priority': str(SD_LOG_LEVEL_MAP[record.levelno]),
|
918
|
+
'tid': str(threading.get_ident()),
|
919
|
+
}
|
920
|
+
|
921
|
+
if (pathname := record.pathname) is not None:
|
922
|
+
fields['code_file'] = pathname
|
923
|
+
if (lineno := record.lineno) is not None:
|
924
|
+
fields['code_lineno'] = str(lineno)
|
925
|
+
if (func_name := record.funcName) is not None:
|
926
|
+
fields['code_func'] = func_name
|
927
|
+
|
928
|
+
for f, a in self.EXTRA_RECORD_ATTRS_BY_JOURNALD_FIELD.items():
|
929
|
+
if (v := getattr(record, a, None)) is not None:
|
930
|
+
fields[f] = str(v)
|
931
|
+
|
932
|
+
return fields
|
933
|
+
|
934
|
+
#
|
935
|
+
|
936
|
+
def emit(self, record: logging.LogRecord) -> None:
|
937
|
+
try:
|
938
|
+
fields = self.make_fields(record)
|
939
|
+
|
940
|
+
if rc := sd_journald_send(**fields):
|
941
|
+
raise RuntimeError(f'{self.__class__.__name__}.emit failed: {rc=}') # noqa
|
942
|
+
|
943
|
+
except RecursionError: # See issue 36272
|
944
|
+
raise
|
945
|
+
|
946
|
+
except Exception: # noqa
|
947
|
+
self.handleError(record)
|
948
|
+
|
949
|
+
|
950
|
+
def journald_log_handler_factory(
|
951
|
+
*,
|
952
|
+
no_tty_check: bool = False,
|
953
|
+
no_fallback: bool = False,
|
954
|
+
) -> logging.Handler:
|
955
|
+
if (
|
956
|
+
sys.platform == 'linux' and
|
957
|
+
(no_tty_check or not sys.stderr.isatty()) and
|
958
|
+
(no_fallback or sd_try_libsystemd() is not None)
|
959
|
+
):
|
960
|
+
return JournaldLogHandler()
|
961
|
+
|
962
|
+
return logging.StreamHandler()
|
963
|
+
|
964
|
+
|
1015
965
|
########################################
|
1016
966
|
# ../../../omlish/lite/logs.py
|
1017
967
|
"""
|
@@ -1099,7 +1049,7 @@ class StandardLogFormatter(logging.Formatter):
|
|
1099
1049
|
if datefmt:
|
1100
1050
|
return ct.strftime(datefmt) # noqa
|
1101
1051
|
else:
|
1102
|
-
t = ct.strftime(
|
1052
|
+
t = ct.strftime('%Y-%m-%d %H:%M:%S')
|
1103
1053
|
return '%s.%03d' % (t, record.msecs)
|
1104
1054
|
|
1105
1055
|
|
@@ -1236,6 +1186,7 @@ def configure_standard_logging(
|
|
1236
1186
|
json: bool = False,
|
1237
1187
|
target: ta.Optional[logging.Logger] = None,
|
1238
1188
|
force: bool = False,
|
1189
|
+
handler_factory: ta.Optional[ta.Callable[[], logging.Handler]] = None,
|
1239
1190
|
) -> ta.Optional[StandardLogHandler]:
|
1240
1191
|
with _locking_logging_module_lock():
|
1241
1192
|
if target is None:
|
@@ -1249,7 +1200,10 @@ def configure_standard_logging(
|
|
1249
1200
|
|
1250
1201
|
#
|
1251
1202
|
|
1252
|
-
|
1203
|
+
if handler_factory is not None:
|
1204
|
+
handler = handler_factory()
|
1205
|
+
else:
|
1206
|
+
handler = logging.StreamHandler()
|
1253
1207
|
|
1254
1208
|
#
|
1255
1209
|
|
@@ -1649,12 +1603,9 @@ class EventCallbacks:
|
|
1649
1603
|
|
1650
1604
|
EVENT_CALLBACKS = EventCallbacks()
|
1651
1605
|
|
1652
|
-
notify_event = EVENT_CALLBACKS.notify
|
1653
|
-
clear_events = EVENT_CALLBACKS.clear
|
1654
|
-
|
1655
1606
|
|
1656
1607
|
class Event(abc.ABC): # noqa
|
1657
|
-
"""Abstract event type
|
1608
|
+
"""Abstract event type."""
|
1658
1609
|
|
1659
1610
|
|
1660
1611
|
class ProcessLogEvent(Event, abc.ABC):
|
@@ -1734,7 +1685,7 @@ class RemoteCommunicationEvent(Event):
|
|
1734
1685
|
|
1735
1686
|
|
1736
1687
|
class SupervisorStateChangeEvent(Event):
|
1737
|
-
"""
|
1688
|
+
"""Abstract class."""
|
1738
1689
|
|
1739
1690
|
def payload(self):
|
1740
1691
|
return ''
|
@@ -1756,7 +1707,7 @@ class EventRejectedEvent: # purposely does not subclass Event
|
|
1756
1707
|
|
1757
1708
|
|
1758
1709
|
class ProcessStateEvent(Event):
|
1759
|
-
"""
|
1710
|
+
"""Abstract class, never raised directly."""
|
1760
1711
|
frm = None
|
1761
1712
|
to = None
|
1762
1713
|
|
@@ -1845,7 +1796,7 @@ class ProcessGroupRemovedEvent(ProcessGroupEvent):
|
|
1845
1796
|
|
1846
1797
|
|
1847
1798
|
class TickEvent(Event):
|
1848
|
-
"""
|
1799
|
+
"""Abstract."""
|
1849
1800
|
|
1850
1801
|
def __init__(self, when, supervisord):
|
1851
1802
|
super().__init__()
|
@@ -1888,39 +1839,273 @@ class EventTypes:
|
|
1888
1839
|
PROCESS_STATE_RUNNING = ProcessStateRunningEvent
|
1889
1840
|
PROCESS_STATE_UNKNOWN = ProcessStateUnknownEvent
|
1890
1841
|
|
1891
|
-
PROCESS_COMMUNICATION = ProcessCommunicationEvent # abstract
|
1892
|
-
PROCESS_COMMUNICATION_STDOUT = ProcessCommunicationStdoutEvent
|
1893
|
-
PROCESS_COMMUNICATION_STDERR = ProcessCommunicationStderrEvent
|
1842
|
+
PROCESS_COMMUNICATION = ProcessCommunicationEvent # abstract
|
1843
|
+
PROCESS_COMMUNICATION_STDOUT = ProcessCommunicationStdoutEvent
|
1844
|
+
PROCESS_COMMUNICATION_STDERR = ProcessCommunicationStderrEvent
|
1845
|
+
|
1846
|
+
PROCESS_LOG = ProcessLogEvent
|
1847
|
+
PROCESS_LOG_STDOUT = ProcessLogStdoutEvent
|
1848
|
+
PROCESS_LOG_STDERR = ProcessLogStderrEvent
|
1849
|
+
|
1850
|
+
REMOTE_COMMUNICATION = RemoteCommunicationEvent
|
1851
|
+
|
1852
|
+
SUPERVISOR_STATE_CHANGE = SupervisorStateChangeEvent # abstract
|
1853
|
+
SUPERVISOR_STATE_CHANGE_RUNNING = SupervisorRunningEvent
|
1854
|
+
SUPERVISOR_STATE_CHANGE_STOPPING = SupervisorStoppingEvent
|
1855
|
+
|
1856
|
+
TICK = TickEvent # abstract
|
1857
|
+
TICK_5 = Tick5Event
|
1858
|
+
TICK_60 = Tick60Event
|
1859
|
+
TICK_3600 = Tick3600Event
|
1860
|
+
|
1861
|
+
PROCESS_GROUP = ProcessGroupEvent # abstract
|
1862
|
+
PROCESS_GROUP_ADDED = ProcessGroupAddedEvent
|
1863
|
+
PROCESS_GROUP_REMOVED = ProcessGroupRemovedEvent
|
1864
|
+
|
1865
|
+
|
1866
|
+
def get_event_name_by_type(requested):
|
1867
|
+
for name, typ in EventTypes.__dict__.items():
|
1868
|
+
if typ is requested:
|
1869
|
+
return name
|
1870
|
+
return None
|
1871
|
+
|
1872
|
+
|
1873
|
+
def register(name, event):
|
1874
|
+
setattr(EventTypes, name, event)
|
1875
|
+
|
1876
|
+
|
1877
|
+
########################################
|
1878
|
+
# ../poller.py
|
1879
|
+
|
1880
|
+
|
1881
|
+
class BasePoller(abc.ABC):
|
1882
|
+
|
1883
|
+
def __init__(self) -> None:
|
1884
|
+
super().__init__()
|
1885
|
+
|
1886
|
+
@abc.abstractmethod
|
1887
|
+
def register_readable(self, fd: int) -> None:
|
1888
|
+
raise NotImplementedError
|
1889
|
+
|
1890
|
+
@abc.abstractmethod
|
1891
|
+
def register_writable(self, fd: int) -> None:
|
1892
|
+
raise NotImplementedError
|
1893
|
+
|
1894
|
+
@abc.abstractmethod
|
1895
|
+
def unregister_readable(self, fd: int) -> None:
|
1896
|
+
raise NotImplementedError
|
1897
|
+
|
1898
|
+
@abc.abstractmethod
|
1899
|
+
def unregister_writable(self, fd: int) -> None:
|
1900
|
+
raise NotImplementedError
|
1901
|
+
|
1902
|
+
@abc.abstractmethod
|
1903
|
+
def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[int], ta.List[int]]:
|
1904
|
+
raise NotImplementedError
|
1905
|
+
|
1906
|
+
def before_daemonize(self) -> None: # noqa
|
1907
|
+
pass
|
1908
|
+
|
1909
|
+
def after_daemonize(self) -> None: # noqa
|
1910
|
+
pass
|
1911
|
+
|
1912
|
+
def close(self) -> None: # noqa
|
1913
|
+
pass
|
1914
|
+
|
1915
|
+
|
1916
|
+
class SelectPoller(BasePoller):
|
1917
|
+
|
1918
|
+
def __init__(self) -> None:
|
1919
|
+
super().__init__()
|
1920
|
+
|
1921
|
+
self._readables: ta.Set[int] = set()
|
1922
|
+
self._writables: ta.Set[int] = set()
|
1923
|
+
|
1924
|
+
def register_readable(self, fd: int) -> None:
|
1925
|
+
self._readables.add(fd)
|
1926
|
+
|
1927
|
+
def register_writable(self, fd: int) -> None:
|
1928
|
+
self._writables.add(fd)
|
1929
|
+
|
1930
|
+
def unregister_readable(self, fd: int) -> None:
|
1931
|
+
self._readables.discard(fd)
|
1932
|
+
|
1933
|
+
def unregister_writable(self, fd: int) -> None:
|
1934
|
+
self._writables.discard(fd)
|
1935
|
+
|
1936
|
+
def unregister_all(self) -> None:
|
1937
|
+
self._readables.clear()
|
1938
|
+
self._writables.clear()
|
1939
|
+
|
1940
|
+
def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[int], ta.List[int]]:
|
1941
|
+
try:
|
1942
|
+
r, w, x = select.select(
|
1943
|
+
self._readables,
|
1944
|
+
self._writables,
|
1945
|
+
[], timeout,
|
1946
|
+
)
|
1947
|
+
except OSError as err:
|
1948
|
+
if err.args[0] == errno.EINTR:
|
1949
|
+
log.debug('EINTR encountered in poll')
|
1950
|
+
return [], []
|
1951
|
+
if err.args[0] == errno.EBADF:
|
1952
|
+
log.debug('EBADF encountered in poll')
|
1953
|
+
self.unregister_all()
|
1954
|
+
return [], []
|
1955
|
+
raise
|
1956
|
+
return r, w
|
1957
|
+
|
1958
|
+
|
1959
|
+
class PollPoller(BasePoller):
|
1960
|
+
_READ = select.POLLIN | select.POLLPRI | select.POLLHUP
|
1961
|
+
_WRITE = select.POLLOUT
|
1962
|
+
|
1963
|
+
def __init__(self) -> None:
|
1964
|
+
super().__init__()
|
1965
|
+
|
1966
|
+
self._poller = select.poll()
|
1967
|
+
self._readables: set[int] = set()
|
1968
|
+
self._writables: set[int] = set()
|
1969
|
+
|
1970
|
+
def register_readable(self, fd: int) -> None:
|
1971
|
+
self._poller.register(fd, self._READ)
|
1972
|
+
self._readables.add(fd)
|
1973
|
+
|
1974
|
+
def register_writable(self, fd: int) -> None:
|
1975
|
+
self._poller.register(fd, self._WRITE)
|
1976
|
+
self._writables.add(fd)
|
1977
|
+
|
1978
|
+
def unregister_readable(self, fd: int) -> None:
|
1979
|
+
self._readables.discard(fd)
|
1980
|
+
self._poller.unregister(fd)
|
1981
|
+
if fd in self._writables:
|
1982
|
+
self._poller.register(fd, self._WRITE)
|
1983
|
+
|
1984
|
+
def unregister_writable(self, fd: int) -> None:
|
1985
|
+
self._writables.discard(fd)
|
1986
|
+
self._poller.unregister(fd)
|
1987
|
+
if fd in self._readables:
|
1988
|
+
self._poller.register(fd, self._READ)
|
1989
|
+
|
1990
|
+
def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[int], ta.List[int]]:
|
1991
|
+
fds = self._poll_fds(timeout) # type: ignore
|
1992
|
+
readables, writables = [], []
|
1993
|
+
for fd, eventmask in fds:
|
1994
|
+
if self._ignore_invalid(fd, eventmask):
|
1995
|
+
continue
|
1996
|
+
if eventmask & self._READ:
|
1997
|
+
readables.append(fd)
|
1998
|
+
if eventmask & self._WRITE:
|
1999
|
+
writables.append(fd)
|
2000
|
+
return readables, writables
|
2001
|
+
|
2002
|
+
def _poll_fds(self, timeout: float) -> ta.List[ta.Tuple[int, int]]:
|
2003
|
+
try:
|
2004
|
+
return self._poller.poll(timeout * 1000)
|
2005
|
+
except OSError as err:
|
2006
|
+
if err.args[0] == errno.EINTR:
|
2007
|
+
log.debug('EINTR encountered in poll')
|
2008
|
+
return []
|
2009
|
+
raise
|
2010
|
+
|
2011
|
+
def _ignore_invalid(self, fd: int, eventmask: int) -> bool:
|
2012
|
+
if eventmask & select.POLLNVAL:
|
2013
|
+
# POLLNVAL means `fd` value is invalid, not open. When a process quits it's `fd`s are closed so there is no
|
2014
|
+
# more reason to keep this `fd` registered If the process restarts it's `fd`s are registered again.
|
2015
|
+
self._poller.unregister(fd)
|
2016
|
+
self._readables.discard(fd)
|
2017
|
+
self._writables.discard(fd)
|
2018
|
+
return True
|
2019
|
+
return False
|
2020
|
+
|
2021
|
+
|
2022
|
+
if sys.platform == 'darwin' or sys.platform.startswith('freebsd'):
|
2023
|
+
class KqueuePoller(BasePoller):
|
2024
|
+
max_events = 1000
|
2025
|
+
|
2026
|
+
def __init__(self) -> None:
|
2027
|
+
super().__init__()
|
2028
|
+
|
2029
|
+
self._kqueue: ta.Optional[ta.Any] = select.kqueue()
|
2030
|
+
self._readables: set[int] = set()
|
2031
|
+
self._writables: set[int] = set()
|
2032
|
+
|
2033
|
+
def register_readable(self, fd: int) -> None:
|
2034
|
+
self._readables.add(fd)
|
2035
|
+
kevent = select.kevent(fd, filter=select.KQ_FILTER_READ, flags=select.KQ_EV_ADD)
|
2036
|
+
self._kqueue_control(fd, kevent)
|
2037
|
+
|
2038
|
+
def register_writable(self, fd: int) -> None:
|
2039
|
+
self._writables.add(fd)
|
2040
|
+
kevent = select.kevent(fd, filter=select.KQ_FILTER_WRITE, flags=select.KQ_EV_ADD)
|
2041
|
+
self._kqueue_control(fd, kevent)
|
2042
|
+
|
2043
|
+
def unregister_readable(self, fd: int) -> None:
|
2044
|
+
kevent = select.kevent(fd, filter=select.KQ_FILTER_READ, flags=select.KQ_EV_DELETE)
|
2045
|
+
self._readables.discard(fd)
|
2046
|
+
self._kqueue_control(fd, kevent)
|
2047
|
+
|
2048
|
+
def unregister_writable(self, fd: int) -> None:
|
2049
|
+
kevent = select.kevent(fd, filter=select.KQ_FILTER_WRITE, flags=select.KQ_EV_DELETE)
|
2050
|
+
self._writables.discard(fd)
|
2051
|
+
self._kqueue_control(fd, kevent)
|
2052
|
+
|
2053
|
+
def _kqueue_control(self, fd: int, kevent: 'select.kevent') -> None:
|
2054
|
+
try:
|
2055
|
+
self._kqueue.control([kevent], 0) # type: ignore
|
2056
|
+
except OSError as error:
|
2057
|
+
if error.errno == errno.EBADF:
|
2058
|
+
log.debug('EBADF encountered in kqueue. Invalid file descriptor %s', fd)
|
2059
|
+
else:
|
2060
|
+
raise
|
2061
|
+
|
2062
|
+
def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[int], ta.List[int]]:
|
2063
|
+
readables, writables = [], [] # type: ignore
|
1894
2064
|
|
1895
|
-
|
1896
|
-
|
1897
|
-
|
2065
|
+
try:
|
2066
|
+
kevents = self._kqueue.control(None, self.max_events, timeout) # type: ignore
|
2067
|
+
except OSError as error:
|
2068
|
+
if error.errno == errno.EINTR:
|
2069
|
+
log.debug('EINTR encountered in poll')
|
2070
|
+
return readables, writables
|
2071
|
+
raise
|
1898
2072
|
|
1899
|
-
|
2073
|
+
for kevent in kevents:
|
2074
|
+
if kevent.filter == select.KQ_FILTER_READ:
|
2075
|
+
readables.append(kevent.ident)
|
2076
|
+
if kevent.filter == select.KQ_FILTER_WRITE:
|
2077
|
+
writables.append(kevent.ident)
|
1900
2078
|
|
1901
|
-
|
1902
|
-
SUPERVISOR_STATE_CHANGE_RUNNING = SupervisorRunningEvent
|
1903
|
-
SUPERVISOR_STATE_CHANGE_STOPPING = SupervisorStoppingEvent
|
2079
|
+
return readables, writables
|
1904
2080
|
|
1905
|
-
|
1906
|
-
|
1907
|
-
TICK_60 = Tick60Event
|
1908
|
-
TICK_3600 = Tick3600Event
|
2081
|
+
def before_daemonize(self) -> None:
|
2082
|
+
self.close()
|
1909
2083
|
|
1910
|
-
|
1911
|
-
|
1912
|
-
|
2084
|
+
def after_daemonize(self) -> None:
|
2085
|
+
self._kqueue = select.kqueue()
|
2086
|
+
for fd in self._readables:
|
2087
|
+
self.register_readable(fd)
|
2088
|
+
for fd in self._writables:
|
2089
|
+
self.register_writable(fd)
|
1913
2090
|
|
2091
|
+
def close(self) -> None:
|
2092
|
+
self._kqueue.close() # type: ignore
|
2093
|
+
self._kqueue = None
|
1914
2094
|
|
1915
|
-
|
1916
|
-
|
1917
|
-
if typ is requested:
|
1918
|
-
return name
|
1919
|
-
return None
|
2095
|
+
else:
|
2096
|
+
KqueuePoller = None
|
1920
2097
|
|
1921
2098
|
|
1922
|
-
|
1923
|
-
|
2099
|
+
Poller: ta.Type[BasePoller]
|
2100
|
+
if (
|
2101
|
+
sys.platform == 'darwin' or sys.platform.startswith('freebsd') and
|
2102
|
+
hasattr(select, 'kqueue') and KqueuePoller is not None
|
2103
|
+
):
|
2104
|
+
Poller = KqueuePoller
|
2105
|
+
elif hasattr(select, 'poll'):
|
2106
|
+
Poller = PollPoller
|
2107
|
+
else:
|
2108
|
+
Poller = SelectPoller
|
1924
2109
|
|
1925
2110
|
|
1926
2111
|
########################################
|
@@ -1973,41 +2158,47 @@ class AbstractSubprocess(abc.ABC):
|
|
1973
2158
|
# ../context.py
|
1974
2159
|
|
1975
2160
|
|
1976
|
-
log = logging.getLogger(__name__)
|
1977
|
-
|
1978
|
-
|
1979
2161
|
class ServerContext(AbstractServerContext):
|
1980
|
-
|
1981
|
-
|
1982
|
-
|
1983
|
-
|
1984
|
-
|
1985
|
-
|
2162
|
+
def __init__(
|
2163
|
+
self,
|
2164
|
+
config: ServerConfig,
|
2165
|
+
*,
|
2166
|
+
epoch: int = 0,
|
2167
|
+
) -> None:
|
1986
2168
|
super().__init__()
|
1987
2169
|
|
1988
2170
|
self._config = config
|
2171
|
+
self._epoch = epoch
|
1989
2172
|
|
1990
2173
|
self._pid_history: ta.Dict[int, AbstractSubprocess] = {}
|
1991
2174
|
self._state: SupervisorState = SupervisorStates.RUNNING
|
1992
2175
|
|
1993
|
-
self.
|
2176
|
+
self._signal_receiver = SignalReceiver()
|
1994
2177
|
|
1995
|
-
self.
|
2178
|
+
self._poller: BasePoller = Poller()
|
1996
2179
|
|
1997
|
-
if
|
1998
|
-
uid = name_to_uid(
|
1999
|
-
self.
|
2000
|
-
self.
|
2180
|
+
if config.user is not None:
|
2181
|
+
uid = name_to_uid(config.user)
|
2182
|
+
self._uid: ta.Optional[int] = uid
|
2183
|
+
self._gid: ta.Optional[int] = gid_for_uid(uid)
|
2001
2184
|
else:
|
2002
|
-
self.
|
2003
|
-
self.
|
2185
|
+
self._uid = None
|
2186
|
+
self._gid = None
|
2004
2187
|
|
2005
|
-
self.
|
2188
|
+
self._unlink_pidfile = False
|
2006
2189
|
|
2007
2190
|
@property
|
2008
2191
|
def config(self) -> ServerConfig:
|
2009
2192
|
return self._config
|
2010
2193
|
|
2194
|
+
@property
|
2195
|
+
def epoch(self) -> int:
|
2196
|
+
return self._epoch
|
2197
|
+
|
2198
|
+
@property
|
2199
|
+
def first(self) -> bool:
|
2200
|
+
return not self._epoch
|
2201
|
+
|
2011
2202
|
@property
|
2012
2203
|
def state(self) -> SupervisorState:
|
2013
2204
|
return self._state
|
@@ -2015,17 +2206,26 @@ class ServerContext(AbstractServerContext):
|
|
2015
2206
|
def set_state(self, state: SupervisorState) -> None:
|
2016
2207
|
self._state = state
|
2017
2208
|
|
2209
|
+
@property
|
2210
|
+
def poller(self) -> BasePoller:
|
2211
|
+
return self._poller
|
2212
|
+
|
2018
2213
|
@property
|
2019
2214
|
def pid_history(self) -> ta.Dict[int, AbstractSubprocess]:
|
2020
2215
|
return self._pid_history
|
2021
2216
|
|
2022
|
-
|
2023
|
-
|
2217
|
+
@property
|
2218
|
+
def uid(self) -> ta.Optional[int]:
|
2219
|
+
return self._uid
|
2220
|
+
|
2221
|
+
@property
|
2222
|
+
def gid(self) -> ta.Optional[int]:
|
2223
|
+
return self._gid
|
2024
2224
|
|
2025
2225
|
##
|
2026
2226
|
|
2027
2227
|
def set_signals(self) -> None:
|
2028
|
-
self.
|
2228
|
+
self._signal_receiver.install(
|
2029
2229
|
signal.SIGTERM,
|
2030
2230
|
signal.SIGINT,
|
2031
2231
|
signal.SIGQUIT,
|
@@ -2136,7 +2336,7 @@ class ServerContext(AbstractServerContext):
|
|
2136
2336
|
))
|
2137
2337
|
|
2138
2338
|
def cleanup(self) -> None:
|
2139
|
-
if self.
|
2339
|
+
if self._unlink_pidfile:
|
2140
2340
|
try_unlink(self.config.pidfile)
|
2141
2341
|
self.poller.close()
|
2142
2342
|
|
@@ -2189,6 +2389,7 @@ class ServerContext(AbstractServerContext):
|
|
2189
2389
|
# Parent
|
2190
2390
|
log.debug('supervisord forked; parent exiting')
|
2191
2391
|
real_exit(0)
|
2392
|
+
|
2192
2393
|
# Child
|
2193
2394
|
log.info('daemonizing the supervisord process')
|
2194
2395
|
if self.config.directory:
|
@@ -2198,11 +2399,15 @@ class ServerContext(AbstractServerContext):
|
|
2198
2399
|
log.critical("can't chdir into %r: %s", self.config.directory, err)
|
2199
2400
|
else:
|
2200
2401
|
log.info('set current directory: %r', self.config.directory)
|
2402
|
+
|
2201
2403
|
os.dup2(0, os.open('/dev/null', os.O_RDONLY))
|
2202
2404
|
os.dup2(1, os.open('/dev/null', os.O_WRONLY))
|
2203
2405
|
os.dup2(2, os.open('/dev/null', os.O_WRONLY))
|
2406
|
+
|
2204
2407
|
os.setsid()
|
2408
|
+
|
2205
2409
|
os.umask(self.config.umask)
|
2410
|
+
|
2206
2411
|
# XXX Stevens, in his Advanced Unix book, section 13.3 (page 417) recommends calling umask(0) and closing unused
|
2207
2412
|
# file descriptors. In his Network Programming book, he additionally recommends ignoring SIGHUP and forking
|
2208
2413
|
# again after the setsid() call, for obscure SVR4 reasons.
|
@@ -2217,7 +2422,7 @@ class ServerContext(AbstractServerContext):
|
|
2217
2422
|
return logfile
|
2218
2423
|
|
2219
2424
|
def get_signal(self) -> ta.Optional[int]:
|
2220
|
-
return self.
|
2425
|
+
return self._signal_receiver.get_signal()
|
2221
2426
|
|
2222
2427
|
def write_pidfile(self) -> None:
|
2223
2428
|
pid = os.getpid()
|
@@ -2227,7 +2432,7 @@ class ServerContext(AbstractServerContext):
|
|
2227
2432
|
except OSError:
|
2228
2433
|
log.critical('could not write pidfile %s', self.config.pidfile)
|
2229
2434
|
else:
|
2230
|
-
self.
|
2435
|
+
self._unlink_pidfile = True
|
2231
2436
|
log.info('supervisord started with pid %s', pid)
|
2232
2437
|
|
2233
2438
|
|
@@ -2280,11 +2485,14 @@ def drop_privileges(user: ta.Union[int, str, None]) -> ta.Optional[str]:
|
|
2280
2485
|
os.setgroups(groups)
|
2281
2486
|
except OSError:
|
2282
2487
|
return 'Could not set groups of effective user'
|
2488
|
+
|
2283
2489
|
try:
|
2284
2490
|
os.setgid(gid)
|
2285
2491
|
except OSError:
|
2286
2492
|
return 'Could not set group id of effective user'
|
2493
|
+
|
2287
2494
|
os.setuid(uid)
|
2495
|
+
|
2288
2496
|
return None
|
2289
2497
|
|
2290
2498
|
|
@@ -2354,9 +2562,6 @@ def check_execv_args(filename, argv, st) -> None:
|
|
2354
2562
|
# ../dispatchers.py
|
2355
2563
|
|
2356
2564
|
|
2357
|
-
log = logging.getLogger(__name__)
|
2358
|
-
|
2359
|
-
|
2360
2565
|
class Dispatcher(abc.ABC):
|
2361
2566
|
|
2362
2567
|
def __init__(self, process: AbstractSubprocess, channel: str, fd: int) -> None:
|
@@ -2513,16 +2718,16 @@ class OutputDispatcher(Dispatcher):
|
|
2513
2718
|
# )
|
2514
2719
|
|
2515
2720
|
def remove_logs(self):
|
2516
|
-
for
|
2517
|
-
if
|
2518
|
-
for handler in
|
2721
|
+
for l in (self._normal_log, self._capture_log):
|
2722
|
+
if l is not None:
|
2723
|
+
for handler in l.handlers:
|
2519
2724
|
handler.remove() # type: ignore
|
2520
2725
|
handler.reopen() # type: ignore
|
2521
2726
|
|
2522
2727
|
def reopen_logs(self):
|
2523
|
-
for
|
2524
|
-
if
|
2525
|
-
for handler in
|
2728
|
+
for l in (self._normal_log, self._capture_log):
|
2729
|
+
if l is not None:
|
2730
|
+
for handler in l.handlers:
|
2526
2731
|
handler.reopen() # type: ignore
|
2527
2732
|
|
2528
2733
|
def _log(self, data):
|
@@ -2545,10 +2750,10 @@ class OutputDispatcher(Dispatcher):
|
|
2545
2750
|
|
2546
2751
|
if self._channel == 'stdout':
|
2547
2752
|
if self._stdout_events_enabled:
|
2548
|
-
|
2753
|
+
EVENT_CALLBACKS.notify(ProcessLogStdoutEvent(self._process, self._process.pid, data))
|
2549
2754
|
|
2550
2755
|
elif self._stderr_events_enabled:
|
2551
|
-
|
2756
|
+
EVENT_CALLBACKS.notify(ProcessLogStderrEvent(self._process, self._process.pid, data))
|
2552
2757
|
|
2553
2758
|
def record_output(self):
|
2554
2759
|
if self._capture_log is None:
|
@@ -2599,7 +2804,7 @@ class OutputDispatcher(Dispatcher):
|
|
2599
2804
|
channel = self._channel
|
2600
2805
|
procname = self._process.config.name
|
2601
2806
|
event = self.event_type(self._process, self._process.pid, data)
|
2602
|
-
|
2807
|
+
EVENT_CALLBACKS.notify(event)
|
2603
2808
|
|
2604
2809
|
log.debug('%r %s emitted a comm event', procname, channel)
|
2605
2810
|
for handler in self._capture_log.handlers:
|
@@ -2660,9 +2865,6 @@ class InputDispatcher(Dispatcher):
|
|
2660
2865
|
# ../process.py
|
2661
2866
|
|
2662
2867
|
|
2663
|
-
log = logging.getLogger(__name__)
|
2664
|
-
|
2665
|
-
|
2666
2868
|
@functools.total_ordering
|
2667
2869
|
class Subprocess(AbstractSubprocess):
|
2668
2870
|
"""A class to manage a subprocess."""
|
@@ -2815,7 +3017,7 @@ class Subprocess(AbstractSubprocess):
|
|
2815
3017
|
event_class = self.event_map.get(new_state)
|
2816
3018
|
if event_class is not None:
|
2817
3019
|
event = event_class(self, old_state, expected)
|
2818
|
-
|
3020
|
+
EVENT_CALLBACKS.notify(event)
|
2819
3021
|
|
2820
3022
|
return True
|
2821
3023
|
|
@@ -2932,6 +3134,7 @@ class Subprocess(AbstractSubprocess):
|
|
2932
3134
|
os.dup2(self._pipes['child_stdout'], 2)
|
2933
3135
|
else:
|
2934
3136
|
os.dup2(self._pipes['child_stderr'], 2)
|
3137
|
+
# FIXME: leave debugger fds
|
2935
3138
|
for i in range(3, self.context.config.minfds):
|
2936
3139
|
close_fd(i)
|
2937
3140
|
|
@@ -2967,7 +3170,7 @@ class Subprocess(AbstractSubprocess):
|
|
2967
3170
|
cwd = self.config.directory
|
2968
3171
|
try:
|
2969
3172
|
if cwd is not None:
|
2970
|
-
os.chdir(cwd)
|
3173
|
+
os.chdir(os.path.expanduser(cwd))
|
2971
3174
|
except OSError as why:
|
2972
3175
|
code = errno.errorcode.get(why.args[0], why.args[0])
|
2973
3176
|
msg = f"couldn't chdir to {cwd}: {code}\n"
|
@@ -3023,7 +3226,7 @@ class Subprocess(AbstractSubprocess):
|
|
3023
3226
|
return self.kill(self.config.stopsignal)
|
3024
3227
|
|
3025
3228
|
def stop_report(self) -> None:
|
3026
|
-
"""
|
3229
|
+
"""Log a 'waiting for x to stop' message with throttling."""
|
3027
3230
|
if self.state == ProcessStates.STOPPING:
|
3028
3231
|
now = time.time()
|
3029
3232
|
|
@@ -3153,7 +3356,7 @@ class Subprocess(AbstractSubprocess):
|
|
3153
3356
|
return None
|
3154
3357
|
|
3155
3358
|
def finish(self, sts: int) -> None:
|
3156
|
-
"""
|
3359
|
+
"""The process was reaped and we need to report and manage its state."""
|
3157
3360
|
|
3158
3361
|
self.drain()
|
3159
3362
|
|
@@ -3234,7 +3437,7 @@ class Subprocess(AbstractSubprocess):
|
|
3234
3437
|
# system that this event was rejected so it can be processed again.
|
3235
3438
|
if self.event is not None:
|
3236
3439
|
# Note: this should only be true if we were in the BUSY state when finish() was called.
|
3237
|
-
|
3440
|
+
EVENT_CALLBACKS.notify(EventRejectedEvent(self, self.event)) # type: ignore
|
3238
3441
|
self.event = None
|
3239
3442
|
|
3240
3443
|
def set_uid(self) -> ta.Optional[str]:
|
@@ -3397,10 +3600,11 @@ class ProcessGroup:
|
|
3397
3600
|
|
3398
3601
|
|
3399
3602
|
########################################
|
3400
|
-
# supervisor.py
|
3603
|
+
# ../supervisor.py
|
3401
3604
|
|
3402
3605
|
|
3403
|
-
|
3606
|
+
def timeslice(period: int, when: float) -> int:
|
3607
|
+
return int(when - (when % period))
|
3404
3608
|
|
3405
3609
|
|
3406
3610
|
class Supervisor:
|
@@ -3415,6 +3619,8 @@ class Supervisor:
|
|
3415
3619
|
self._stopping = False # set after we detect that we are handling a stop request
|
3416
3620
|
self._last_shutdown_report = 0. # throttle for delayed process error reports at stop
|
3417
3621
|
|
3622
|
+
#
|
3623
|
+
|
3418
3624
|
@property
|
3419
3625
|
def context(self) -> ServerContext:
|
3420
3626
|
return self._context
|
@@ -3422,47 +3628,14 @@ class Supervisor:
|
|
3422
3628
|
def get_state(self) -> SupervisorState:
|
3423
3629
|
return self._context.state
|
3424
3630
|
|
3425
|
-
|
3426
|
-
if not self._context.first:
|
3427
|
-
# prevent crash on libdispatch-based systems, at least for the first request
|
3428
|
-
self._context.cleanup_fds()
|
3429
|
-
|
3430
|
-
self._context.set_uid_or_exit()
|
3431
|
-
|
3432
|
-
if self._context.first:
|
3433
|
-
self._context.set_rlimits_or_exit()
|
3434
|
-
|
3435
|
-
# this sets the options.logger object delay logger instantiation until after setuid
|
3436
|
-
if not self._context.config.nocleanup:
|
3437
|
-
# clean up old automatic logs
|
3438
|
-
self._context.clear_auto_child_logdir()
|
3439
|
-
|
3440
|
-
self.run()
|
3441
|
-
|
3442
|
-
def run(self) -> None:
|
3443
|
-
self._process_groups = {} # clear
|
3444
|
-
self._stop_groups = None # clear
|
3445
|
-
|
3446
|
-
clear_events()
|
3447
|
-
|
3448
|
-
try:
|
3449
|
-
for config in self._context.config.groups or []:
|
3450
|
-
self.add_process_group(config)
|
3451
|
-
|
3452
|
-
self._context.set_signals()
|
3453
|
-
|
3454
|
-
if not self._context.config.nodaemon and self._context.first:
|
3455
|
-
self._context.daemonize()
|
3456
|
-
|
3457
|
-
# writing pid file needs to come *after* daemonizing or pid will be wrong
|
3458
|
-
self._context.write_pidfile()
|
3459
|
-
|
3460
|
-
self.runforever()
|
3631
|
+
#
|
3461
3632
|
|
3462
|
-
|
3463
|
-
|
3633
|
+
class DiffToActive(ta.NamedTuple):
|
3634
|
+
added: ta.List[ProcessGroupConfig]
|
3635
|
+
changed: ta.List[ProcessGroupConfig]
|
3636
|
+
removed: ta.List[ProcessGroupConfig]
|
3464
3637
|
|
3465
|
-
def diff_to_active(self):
|
3638
|
+
def diff_to_active(self) -> DiffToActive:
|
3466
3639
|
new = self._context.config.groups or []
|
3467
3640
|
cur = [group.config for group in self._process_groups.values()]
|
3468
3641
|
|
@@ -3474,7 +3647,7 @@ class Supervisor:
|
|
3474
3647
|
|
3475
3648
|
changed = [cand for cand in new if cand != curdict.get(cand.name, cand)]
|
3476
3649
|
|
3477
|
-
return added, changed, removed
|
3650
|
+
return Supervisor.DiffToActive(added, changed, removed)
|
3478
3651
|
|
3479
3652
|
def add_process_group(self, config: ProcessGroupConfig) -> bool:
|
3480
3653
|
name = config.name
|
@@ -3484,7 +3657,7 @@ class Supervisor:
|
|
3484
3657
|
group = self._process_groups[name] = ProcessGroup(config, self._context)
|
3485
3658
|
group.after_setuid()
|
3486
3659
|
|
3487
|
-
|
3660
|
+
EVENT_CALLBACKS.notify(ProcessGroupAddedEvent(name))
|
3488
3661
|
return True
|
3489
3662
|
|
3490
3663
|
def remove_process_group(self, name: str) -> bool:
|
@@ -3495,7 +3668,7 @@ class Supervisor:
|
|
3495
3668
|
|
3496
3669
|
del self._process_groups[name]
|
3497
3670
|
|
3498
|
-
|
3671
|
+
EVENT_CALLBACKS.notify(ProcessGroupRemovedEvent(name))
|
3499
3672
|
return True
|
3500
3673
|
|
3501
3674
|
def get_process_map(self) -> ta.Dict[int, Dispatcher]:
|
@@ -3524,6 +3697,72 @@ class Supervisor:
|
|
3524
3697
|
|
3525
3698
|
return unstopped
|
3526
3699
|
|
3700
|
+
#
|
3701
|
+
|
3702
|
+
def main(self) -> None:
|
3703
|
+
self.setup()
|
3704
|
+
self.run()
|
3705
|
+
|
3706
|
+
@cached_nullary
|
3707
|
+
def setup(self) -> None:
|
3708
|
+
if not self._context.first:
|
3709
|
+
# prevent crash on libdispatch-based systems, at least for the first request
|
3710
|
+
self._context.cleanup_fds()
|
3711
|
+
|
3712
|
+
self._context.set_uid_or_exit()
|
3713
|
+
|
3714
|
+
if self._context.first:
|
3715
|
+
self._context.set_rlimits_or_exit()
|
3716
|
+
|
3717
|
+
# this sets the options.logger object delay logger instantiation until after setuid
|
3718
|
+
if not self._context.config.nocleanup:
|
3719
|
+
# clean up old automatic logs
|
3720
|
+
self._context.clear_auto_child_logdir()
|
3721
|
+
|
3722
|
+
def run(
|
3723
|
+
self,
|
3724
|
+
*,
|
3725
|
+
callback: ta.Optional[ta.Callable[['Supervisor'], bool]] = None,
|
3726
|
+
) -> None:
|
3727
|
+
self._process_groups = {} # clear
|
3728
|
+
self._stop_groups = None # clear
|
3729
|
+
|
3730
|
+
EVENT_CALLBACKS.clear()
|
3731
|
+
|
3732
|
+
try:
|
3733
|
+
for config in self._context.config.groups or []:
|
3734
|
+
self.add_process_group(config)
|
3735
|
+
|
3736
|
+
self._context.set_signals()
|
3737
|
+
|
3738
|
+
if not self._context.config.nodaemon and self._context.first:
|
3739
|
+
self._context.daemonize()
|
3740
|
+
|
3741
|
+
# writing pid file needs to come *after* daemonizing or pid will be wrong
|
3742
|
+
self._context.write_pidfile()
|
3743
|
+
|
3744
|
+
EVENT_CALLBACKS.notify(SupervisorRunningEvent())
|
3745
|
+
|
3746
|
+
while True:
|
3747
|
+
if callback is not None and not callback(self):
|
3748
|
+
break
|
3749
|
+
|
3750
|
+
self._run_once()
|
3751
|
+
|
3752
|
+
finally:
|
3753
|
+
self._context.cleanup()
|
3754
|
+
|
3755
|
+
#
|
3756
|
+
|
3757
|
+
def _run_once(self) -> None:
|
3758
|
+
self._poll()
|
3759
|
+
self._reap()
|
3760
|
+
self._handle_signal()
|
3761
|
+
self._tick()
|
3762
|
+
|
3763
|
+
if self._context.state < SupervisorStates.RUNNING:
|
3764
|
+
self._ordered_stop_groups_phase_2()
|
3765
|
+
|
3527
3766
|
def _ordered_stop_groups_phase_1(self) -> None:
|
3528
3767
|
if self._stop_groups:
|
3529
3768
|
# stop the last group (the one with the "highest" priority)
|
@@ -3540,110 +3779,77 @@ class Supervisor:
|
|
3540
3779
|
# down, so push it back on to the end of the stop group queue
|
3541
3780
|
self._stop_groups.append(group)
|
3542
3781
|
|
3543
|
-
def
|
3544
|
-
|
3545
|
-
|
3546
|
-
|
3547
|
-
while True:
|
3548
|
-
combined_map = {}
|
3549
|
-
combined_map.update(self.get_process_map())
|
3782
|
+
def _poll(self) -> None:
|
3783
|
+
combined_map = {}
|
3784
|
+
combined_map.update(self.get_process_map())
|
3550
3785
|
|
3551
|
-
|
3552
|
-
|
3786
|
+
pgroups = list(self._process_groups.values())
|
3787
|
+
pgroups.sort()
|
3553
3788
|
|
3554
|
-
|
3555
|
-
|
3556
|
-
|
3557
|
-
|
3558
|
-
|
3559
|
-
|
3789
|
+
if self._context.state < SupervisorStates.RUNNING:
|
3790
|
+
if not self._stopping:
|
3791
|
+
# first time, set the stopping flag, do a notification and set stop_groups
|
3792
|
+
self._stopping = True
|
3793
|
+
self._stop_groups = pgroups[:]
|
3794
|
+
EVENT_CALLBACKS.notify(SupervisorStoppingEvent())
|
3560
3795
|
|
3561
|
-
|
3796
|
+
self._ordered_stop_groups_phase_1()
|
3562
3797
|
|
3563
|
-
|
3564
|
-
|
3565
|
-
|
3798
|
+
if not self.shutdown_report():
|
3799
|
+
# if there are no unstopped processes (we're done killing everything), it's OK to shutdown or reload
|
3800
|
+
raise ExitNow
|
3566
3801
|
|
3567
|
-
|
3568
|
-
|
3569
|
-
|
3570
|
-
|
3571
|
-
|
3802
|
+
for fd, dispatcher in combined_map.items():
|
3803
|
+
if dispatcher.readable():
|
3804
|
+
self._context.poller.register_readable(fd)
|
3805
|
+
if dispatcher.writable():
|
3806
|
+
self._context.poller.register_writable(fd)
|
3572
3807
|
|
3573
|
-
|
3808
|
+
timeout = 1 # this cannot be fewer than the smallest TickEvent (5)
|
3809
|
+
r, w = self._context.poller.poll(timeout)
|
3574
3810
|
|
3575
|
-
|
3576
|
-
|
3577
|
-
|
3578
|
-
|
3579
|
-
|
3580
|
-
|
3581
|
-
|
3582
|
-
self._context.poller.unregister_readable(fd)
|
3583
|
-
except ExitNow:
|
3584
|
-
raise
|
3585
|
-
except Exception: # noqa
|
3586
|
-
combined_map[fd].handle_error()
|
3587
|
-
else:
|
3588
|
-
# if the fd is not in combined_map, we should unregister it. otherwise, it will be polled every
|
3589
|
-
# time, which may cause 100% cpu usage
|
3590
|
-
log.debug('unexpected read event from fd %r', fd)
|
3591
|
-
try:
|
3811
|
+
for fd in r:
|
3812
|
+
if fd in combined_map:
|
3813
|
+
try:
|
3814
|
+
dispatcher = combined_map[fd]
|
3815
|
+
log.debug('read event caused by %r', dispatcher)
|
3816
|
+
dispatcher.handle_read_event()
|
3817
|
+
if not dispatcher.readable():
|
3592
3818
|
self._context.poller.unregister_readable(fd)
|
3593
|
-
|
3594
|
-
|
3819
|
+
except ExitNow:
|
3820
|
+
raise
|
3821
|
+
except Exception: # noqa
|
3822
|
+
combined_map[fd].handle_error()
|
3823
|
+
else:
|
3824
|
+
# if the fd is not in combined_map, we should unregister it. otherwise, it will be polled every
|
3825
|
+
# time, which may cause 100% cpu usage
|
3826
|
+
log.debug('unexpected read event from fd %r', fd)
|
3827
|
+
try:
|
3828
|
+
self._context.poller.unregister_readable(fd)
|
3829
|
+
except Exception: # noqa
|
3830
|
+
pass
|
3595
3831
|
|
3596
|
-
|
3597
|
-
|
3598
|
-
|
3599
|
-
|
3600
|
-
|
3601
|
-
|
3602
|
-
|
3603
|
-
self._context.poller.unregister_writable(fd)
|
3604
|
-
except ExitNow:
|
3605
|
-
raise
|
3606
|
-
except Exception: # noqa
|
3607
|
-
combined_map[fd].handle_error()
|
3608
|
-
else:
|
3609
|
-
log.debug('unexpected write event from fd %r', fd)
|
3610
|
-
try:
|
3832
|
+
for fd in w:
|
3833
|
+
if fd in combined_map:
|
3834
|
+
try:
|
3835
|
+
dispatcher = combined_map[fd]
|
3836
|
+
log.debug('write event caused by %r', dispatcher)
|
3837
|
+
dispatcher.handle_write_event()
|
3838
|
+
if not dispatcher.writable():
|
3611
3839
|
self._context.poller.unregister_writable(fd)
|
3612
|
-
|
3613
|
-
|
3614
|
-
|
3615
|
-
|
3616
|
-
|
3617
|
-
|
3618
|
-
|
3619
|
-
|
3620
|
-
|
3621
|
-
|
3622
|
-
if self._context.state < SupervisorStates.RUNNING:
|
3623
|
-
self._ordered_stop_groups_phase_2()
|
3624
|
-
|
3625
|
-
if self._context.test:
|
3626
|
-
break
|
3627
|
-
|
3628
|
-
def _tick(self, now: ta.Optional[float] = None) -> None:
|
3629
|
-
"""Send one or more 'tick' events when the timeslice related to the period for the event type rolls over"""
|
3630
|
-
|
3631
|
-
if now is None:
|
3632
|
-
# now won't be None in unit tests
|
3633
|
-
now = time.time()
|
3634
|
-
|
3635
|
-
for event in TICK_EVENTS:
|
3636
|
-
period = event.period # type: ignore
|
3637
|
-
|
3638
|
-
last_tick = self._ticks.get(period)
|
3639
|
-
if last_tick is None:
|
3640
|
-
# we just started up
|
3641
|
-
last_tick = self._ticks[period] = timeslice(period, now)
|
3840
|
+
except ExitNow:
|
3841
|
+
raise
|
3842
|
+
except Exception: # noqa
|
3843
|
+
combined_map[fd].handle_error()
|
3844
|
+
else:
|
3845
|
+
log.debug('unexpected write event from fd %r', fd)
|
3846
|
+
try:
|
3847
|
+
self._context.poller.unregister_writable(fd)
|
3848
|
+
except Exception: # noqa
|
3849
|
+
pass
|
3642
3850
|
|
3643
|
-
|
3644
|
-
|
3645
|
-
self._ticks[period] = this_tick
|
3646
|
-
notify_event(event(this_tick, self))
|
3851
|
+
for group in pgroups:
|
3852
|
+
group.transition()
|
3647
3853
|
|
3648
3854
|
def _reap(self, *, once: bool = False, depth: int = 0) -> None:
|
3649
3855
|
if depth >= 100:
|
@@ -3693,50 +3899,77 @@ class Supervisor:
|
|
3693
3899
|
else:
|
3694
3900
|
log.debug('received %s indicating nothing', signame(sig))
|
3695
3901
|
|
3902
|
+
def _tick(self, now: ta.Optional[float] = None) -> None:
|
3903
|
+
"""Send one or more 'tick' events when the timeslice related to the period for the event type rolls over"""
|
3904
|
+
|
3905
|
+
if now is None:
|
3906
|
+
# now won't be None in unit tests
|
3907
|
+
now = time.time()
|
3696
3908
|
|
3697
|
-
|
3698
|
-
|
3909
|
+
for event in TICK_EVENTS:
|
3910
|
+
period = event.period # type: ignore
|
3911
|
+
|
3912
|
+
last_tick = self._ticks.get(period)
|
3913
|
+
if last_tick is None:
|
3914
|
+
# we just started up
|
3915
|
+
last_tick = self._ticks[period] = timeslice(period, now)
|
3916
|
+
|
3917
|
+
this_tick = timeslice(period, now)
|
3918
|
+
if this_tick != last_tick:
|
3919
|
+
self._ticks[period] = this_tick
|
3920
|
+
EVENT_CALLBACKS.notify(event(this_tick, self))
|
3921
|
+
|
3922
|
+
|
3923
|
+
########################################
|
3924
|
+
# main.py
|
3699
3925
|
|
3700
3926
|
|
3701
|
-
def main(
|
3927
|
+
def main(
|
3928
|
+
argv: ta.Optional[ta.Sequence[str]] = None,
|
3929
|
+
*,
|
3930
|
+
no_logging: bool = False,
|
3931
|
+
) -> None:
|
3702
3932
|
import argparse
|
3703
3933
|
|
3704
3934
|
parser = argparse.ArgumentParser()
|
3705
3935
|
parser.add_argument('config_file', metavar='config-file')
|
3706
|
-
|
3936
|
+
parser.add_argument('--no-journald', action='store_true')
|
3937
|
+
args = parser.parse_args(argv)
|
3707
3938
|
|
3708
|
-
|
3939
|
+
#
|
3709
3940
|
|
3710
3941
|
if not (cf := args.config_file):
|
3711
3942
|
raise RuntimeError('No config file specified')
|
3712
3943
|
|
3944
|
+
if not no_logging:
|
3945
|
+
configure_standard_logging(
|
3946
|
+
'INFO',
|
3947
|
+
handler_factory=journald_log_handler_factory if not args.no_journald else None,
|
3948
|
+
)
|
3949
|
+
|
3950
|
+
#
|
3951
|
+
|
3713
3952
|
# if we hup, restart by making a new Supervisor()
|
3714
|
-
|
3715
|
-
while True:
|
3953
|
+
for epoch in itertools.count():
|
3716
3954
|
with open(cf) as f:
|
3717
3955
|
config_src = f.read()
|
3956
|
+
|
3718
3957
|
config_dct = json.loads(config_src)
|
3719
3958
|
config: ServerConfig = unmarshal_obj(config_dct, ServerConfig)
|
3720
3959
|
|
3721
3960
|
context = ServerContext(
|
3722
3961
|
config,
|
3962
|
+
epoch=epoch,
|
3723
3963
|
)
|
3724
3964
|
|
3725
|
-
|
3726
|
-
|
3727
|
-
|
3728
|
-
|
3729
|
-
|
3730
|
-
if test or (context.state < SupervisorStates.RESTARTING):
|
3731
|
-
break
|
3732
|
-
|
3965
|
+
supervisor = Supervisor(context)
|
3966
|
+
try:
|
3967
|
+
supervisor.main()
|
3968
|
+
except ExitNow:
|
3969
|
+
pass
|
3733
3970
|
|
3734
|
-
|
3735
|
-
|
3736
|
-
try:
|
3737
|
-
d.main()
|
3738
|
-
except ExitNow:
|
3739
|
-
pass
|
3971
|
+
if context.state < SupervisorStates.RESTARTING:
|
3972
|
+
break
|
3740
3973
|
|
3741
3974
|
|
3742
3975
|
if __name__ == '__main__':
|