ominfra 0.0.0.dev118__py3-none-any.whl → 0.0.0.dev120__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- ominfra/deploy/_executor.py +6 -2
- ominfra/deploy/poly/_main.py +6 -2
- ominfra/journald/fields.py +187 -0
- ominfra/journald/tailer.py +375 -312
- ominfra/pyremote/_runcommands.py +6 -2
- ominfra/scripts/journald2aws.py +381 -314
- ominfra/scripts/supervisor.py +697 -464
- ominfra/supervisor/__main__.py +1 -1
- ominfra/supervisor/context.py +50 -25
- ominfra/supervisor/dispatchers.py +12 -13
- ominfra/supervisor/events.py +4 -7
- ominfra/supervisor/main.py +68 -0
- ominfra/supervisor/poller.py +1 -3
- ominfra/supervisor/process.py +10 -11
- ominfra/supervisor/supervisor.py +161 -193
- {ominfra-0.0.0.dev118.dist-info → ominfra-0.0.0.dev120.dist-info}/METADATA +3 -3
- {ominfra-0.0.0.dev118.dist-info → ominfra-0.0.0.dev120.dist-info}/RECORD +21 -19
- {ominfra-0.0.0.dev118.dist-info → ominfra-0.0.0.dev120.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev118.dist-info → ominfra-0.0.0.dev120.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev118.dist-info → ominfra-0.0.0.dev120.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev118.dist-info → ominfra-0.0.0.dev120.dist-info}/top_level.txt +0 -0
ominfra/scripts/supervisor.py
CHANGED
@@ -2,12 +2,13 @@
|
|
2
2
|
# noinspection DuplicatedCode
|
3
3
|
# @omlish-lite
|
4
4
|
# @omlish-script
|
5
|
-
# @omlish-amalg-output ../supervisor/
|
6
|
-
# ruff: noqa: N802 UP006 UP007 UP036
|
5
|
+
# @omlish-amalg-output ../supervisor/main.py
|
6
|
+
# ruff: noqa: N802 UP006 UP007 UP012 UP036
|
7
7
|
import abc
|
8
8
|
import base64
|
9
9
|
import collections.abc
|
10
10
|
import contextlib
|
11
|
+
import ctypes as ct
|
11
12
|
import dataclasses as dc
|
12
13
|
import datetime
|
13
14
|
import decimal
|
@@ -17,9 +18,11 @@ import fcntl
|
|
17
18
|
import fractions
|
18
19
|
import functools
|
19
20
|
import grp
|
21
|
+
import itertools
|
20
22
|
import json
|
21
23
|
import logging
|
22
24
|
import os
|
25
|
+
import os.path
|
23
26
|
import pwd
|
24
27
|
import re
|
25
28
|
import resource
|
@@ -28,6 +31,7 @@ import shlex
|
|
28
31
|
import signal
|
29
32
|
import stat
|
30
33
|
import sys
|
34
|
+
import syslog
|
31
35
|
import tempfile
|
32
36
|
import threading
|
33
37
|
import time
|
@@ -466,240 +470,28 @@ class NoPermissionError(ProcessError):
|
|
466
470
|
|
467
471
|
|
468
472
|
########################################
|
469
|
-
#
|
470
|
-
|
471
|
-
|
472
|
-
log = logging.getLogger(__name__)
|
473
|
-
|
474
|
-
|
475
|
-
class BasePoller(abc.ABC):
|
476
|
-
|
477
|
-
def __init__(self) -> None:
|
478
|
-
super().__init__()
|
479
|
-
|
480
|
-
@abc.abstractmethod
|
481
|
-
def register_readable(self, fd: int) -> None:
|
482
|
-
raise NotImplementedError
|
483
|
-
|
484
|
-
@abc.abstractmethod
|
485
|
-
def register_writable(self, fd: int) -> None:
|
486
|
-
raise NotImplementedError
|
487
|
-
|
488
|
-
@abc.abstractmethod
|
489
|
-
def unregister_readable(self, fd: int) -> None:
|
490
|
-
raise NotImplementedError
|
491
|
-
|
492
|
-
@abc.abstractmethod
|
493
|
-
def unregister_writable(self, fd: int) -> None:
|
494
|
-
raise NotImplementedError
|
495
|
-
|
496
|
-
@abc.abstractmethod
|
497
|
-
def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[int], ta.List[int]]:
|
498
|
-
raise NotImplementedError
|
499
|
-
|
500
|
-
def before_daemonize(self) -> None: # noqa
|
501
|
-
pass
|
502
|
-
|
503
|
-
def after_daemonize(self) -> None: # noqa
|
504
|
-
pass
|
505
|
-
|
506
|
-
def close(self) -> None: # noqa
|
507
|
-
pass
|
508
|
-
|
509
|
-
|
510
|
-
class SelectPoller(BasePoller):
|
511
|
-
|
512
|
-
def __init__(self) -> None:
|
513
|
-
super().__init__()
|
514
|
-
|
515
|
-
self._readables: ta.Set[int] = set()
|
516
|
-
self._writables: ta.Set[int] = set()
|
517
|
-
|
518
|
-
def register_readable(self, fd: int) -> None:
|
519
|
-
self._readables.add(fd)
|
520
|
-
|
521
|
-
def register_writable(self, fd: int) -> None:
|
522
|
-
self._writables.add(fd)
|
523
|
-
|
524
|
-
def unregister_readable(self, fd: int) -> None:
|
525
|
-
self._readables.discard(fd)
|
526
|
-
|
527
|
-
def unregister_writable(self, fd: int) -> None:
|
528
|
-
self._writables.discard(fd)
|
529
|
-
|
530
|
-
def unregister_all(self) -> None:
|
531
|
-
self._readables.clear()
|
532
|
-
self._writables.clear()
|
473
|
+
# ../../../omlish/lite/cached.py
|
533
474
|
|
534
|
-
def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[int], ta.List[int]]:
|
535
|
-
try:
|
536
|
-
r, w, x = select.select(
|
537
|
-
self._readables,
|
538
|
-
self._writables,
|
539
|
-
[], timeout,
|
540
|
-
)
|
541
|
-
except OSError as err:
|
542
|
-
if err.args[0] == errno.EINTR:
|
543
|
-
log.debug('EINTR encountered in poll')
|
544
|
-
return [], []
|
545
|
-
if err.args[0] == errno.EBADF:
|
546
|
-
log.debug('EBADF encountered in poll')
|
547
|
-
self.unregister_all()
|
548
|
-
return [], []
|
549
|
-
raise
|
550
|
-
return r, w
|
551
|
-
|
552
|
-
|
553
|
-
class PollPoller(BasePoller):
|
554
|
-
_READ = select.POLLIN | select.POLLPRI | select.POLLHUP
|
555
|
-
_WRITE = select.POLLOUT
|
556
475
|
|
557
|
-
|
476
|
+
class _cached_nullary: # noqa
|
477
|
+
def __init__(self, fn):
|
558
478
|
super().__init__()
|
479
|
+
self._fn = fn
|
480
|
+
self._value = self._missing = object()
|
481
|
+
functools.update_wrapper(self, fn)
|
559
482
|
|
560
|
-
|
561
|
-
self.
|
562
|
-
|
563
|
-
|
564
|
-
def register_readable(self, fd: int) -> None:
|
565
|
-
self._poller.register(fd, self._READ)
|
566
|
-
self._readables.add(fd)
|
567
|
-
|
568
|
-
def register_writable(self, fd: int) -> None:
|
569
|
-
self._poller.register(fd, self._WRITE)
|
570
|
-
self._writables.add(fd)
|
483
|
+
def __call__(self, *args, **kwargs): # noqa
|
484
|
+
if self._value is self._missing:
|
485
|
+
self._value = self._fn()
|
486
|
+
return self._value
|
571
487
|
|
572
|
-
def
|
573
|
-
self.
|
574
|
-
|
575
|
-
if fd in self._writables:
|
576
|
-
self._poller.register(fd, self._WRITE)
|
488
|
+
def __get__(self, instance, owner): # noqa
|
489
|
+
bound = instance.__dict__[self._fn.__name__] = self.__class__(self._fn.__get__(instance, owner))
|
490
|
+
return bound
|
577
491
|
|
578
|
-
def unregister_writable(self, fd: int) -> None:
|
579
|
-
self._writables.discard(fd)
|
580
|
-
self._poller.unregister(fd)
|
581
|
-
if fd in self._readables:
|
582
|
-
self._poller.register(fd, self._READ)
|
583
492
|
|
584
|
-
|
585
|
-
|
586
|
-
readables, writables = [], []
|
587
|
-
for fd, eventmask in fds:
|
588
|
-
if self._ignore_invalid(fd, eventmask):
|
589
|
-
continue
|
590
|
-
if eventmask & self._READ:
|
591
|
-
readables.append(fd)
|
592
|
-
if eventmask & self._WRITE:
|
593
|
-
writables.append(fd)
|
594
|
-
return readables, writables
|
595
|
-
|
596
|
-
def _poll_fds(self, timeout: float) -> ta.List[ta.Tuple[int, int]]:
|
597
|
-
try:
|
598
|
-
return self._poller.poll(timeout * 1000)
|
599
|
-
except OSError as err:
|
600
|
-
if err.args[0] == errno.EINTR:
|
601
|
-
log.debug('EINTR encountered in poll')
|
602
|
-
return []
|
603
|
-
raise
|
604
|
-
|
605
|
-
def _ignore_invalid(self, fd: int, eventmask: int) -> bool:
|
606
|
-
if eventmask & select.POLLNVAL:
|
607
|
-
# POLLNVAL means `fd` value is invalid, not open. When a process quits it's `fd`s are closed so there is no
|
608
|
-
# more reason to keep this `fd` registered If the process restarts it's `fd`s are registered again.
|
609
|
-
self._poller.unregister(fd)
|
610
|
-
self._readables.discard(fd)
|
611
|
-
self._writables.discard(fd)
|
612
|
-
return True
|
613
|
-
return False
|
614
|
-
|
615
|
-
|
616
|
-
if sys.platform == 'darwin' or sys.platform.startswith('freebsd'):
|
617
|
-
class KqueuePoller(BasePoller):
|
618
|
-
max_events = 1000
|
619
|
-
|
620
|
-
def __init__(self) -> None:
|
621
|
-
super().__init__()
|
622
|
-
|
623
|
-
self._kqueue: ta.Optional[ta.Any] = select.kqueue()
|
624
|
-
self._readables: set[int] = set()
|
625
|
-
self._writables: set[int] = set()
|
626
|
-
|
627
|
-
def register_readable(self, fd: int) -> None:
|
628
|
-
self._readables.add(fd)
|
629
|
-
kevent = select.kevent(fd, filter=select.KQ_FILTER_READ, flags=select.KQ_EV_ADD)
|
630
|
-
self._kqueue_control(fd, kevent)
|
631
|
-
|
632
|
-
def register_writable(self, fd: int) -> None:
|
633
|
-
self._writables.add(fd)
|
634
|
-
kevent = select.kevent(fd, filter=select.KQ_FILTER_WRITE, flags=select.KQ_EV_ADD)
|
635
|
-
self._kqueue_control(fd, kevent)
|
636
|
-
|
637
|
-
def unregister_readable(self, fd: int) -> None:
|
638
|
-
kevent = select.kevent(fd, filter=select.KQ_FILTER_READ, flags=select.KQ_EV_DELETE)
|
639
|
-
self._readables.discard(fd)
|
640
|
-
self._kqueue_control(fd, kevent)
|
641
|
-
|
642
|
-
def unregister_writable(self, fd: int) -> None:
|
643
|
-
kevent = select.kevent(fd, filter=select.KQ_FILTER_WRITE, flags=select.KQ_EV_DELETE)
|
644
|
-
self._writables.discard(fd)
|
645
|
-
self._kqueue_control(fd, kevent)
|
646
|
-
|
647
|
-
def _kqueue_control(self, fd: int, kevent: 'select.kevent') -> None:
|
648
|
-
try:
|
649
|
-
self._kqueue.control([kevent], 0) # type: ignore
|
650
|
-
except OSError as error:
|
651
|
-
if error.errno == errno.EBADF:
|
652
|
-
log.debug('EBADF encountered in kqueue. Invalid file descriptor %s', fd)
|
653
|
-
else:
|
654
|
-
raise
|
655
|
-
|
656
|
-
def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[int], ta.List[int]]:
|
657
|
-
readables, writables = [], [] # type: ignore
|
658
|
-
|
659
|
-
try:
|
660
|
-
kevents = self._kqueue.control(None, self.max_events, timeout) # type: ignore
|
661
|
-
except OSError as error:
|
662
|
-
if error.errno == errno.EINTR:
|
663
|
-
log.debug('EINTR encountered in poll')
|
664
|
-
return readables, writables
|
665
|
-
raise
|
666
|
-
|
667
|
-
for kevent in kevents:
|
668
|
-
if kevent.filter == select.KQ_FILTER_READ:
|
669
|
-
readables.append(kevent.ident)
|
670
|
-
if kevent.filter == select.KQ_FILTER_WRITE:
|
671
|
-
writables.append(kevent.ident)
|
672
|
-
|
673
|
-
return readables, writables
|
674
|
-
|
675
|
-
def before_daemonize(self) -> None:
|
676
|
-
self.close()
|
677
|
-
|
678
|
-
def after_daemonize(self) -> None:
|
679
|
-
self._kqueue = select.kqueue()
|
680
|
-
for fd in self._readables:
|
681
|
-
self.register_readable(fd)
|
682
|
-
for fd in self._writables:
|
683
|
-
self.register_writable(fd)
|
684
|
-
|
685
|
-
def close(self) -> None:
|
686
|
-
self._kqueue.close() # type: ignore
|
687
|
-
self._kqueue = None
|
688
|
-
|
689
|
-
else:
|
690
|
-
KqueuePoller = None
|
691
|
-
|
692
|
-
|
693
|
-
Poller: ta.Type[BasePoller]
|
694
|
-
if (
|
695
|
-
sys.platform == 'darwin' or sys.platform.startswith('freebsd') and
|
696
|
-
hasattr(select, 'kqueue') and KqueuePoller is not None
|
697
|
-
):
|
698
|
-
Poller = KqueuePoller
|
699
|
-
elif hasattr(select, 'poll'):
|
700
|
-
Poller = PollPoller
|
701
|
-
else:
|
702
|
-
Poller = SelectPoller
|
493
|
+
def cached_nullary(fn: ta.Callable[..., T]) -> ta.Callable[..., T]:
|
494
|
+
return _cached_nullary(fn)
|
703
495
|
|
704
496
|
|
705
497
|
########################################
|
@@ -1012,6 +804,164 @@ def get_supervisor_state_description(code: SupervisorState) -> str:
|
|
1012
804
|
return check_not_none(_supervisor_states_by_code.get(code))
|
1013
805
|
|
1014
806
|
|
807
|
+
########################################
|
808
|
+
# ../../../omlish/lite/journald.py
|
809
|
+
|
810
|
+
|
811
|
+
##
|
812
|
+
|
813
|
+
|
814
|
+
class sd_iovec(ct.Structure): # noqa
|
815
|
+
pass
|
816
|
+
|
817
|
+
|
818
|
+
sd_iovec._fields_ = [
|
819
|
+
('iov_base', ct.c_void_p), # Pointer to data.
|
820
|
+
('iov_len', ct.c_size_t), # Length of data.
|
821
|
+
]
|
822
|
+
|
823
|
+
|
824
|
+
##
|
825
|
+
|
826
|
+
|
827
|
+
@cached_nullary
|
828
|
+
def sd_libsystemd() -> ta.Any:
|
829
|
+
lib = ct.CDLL('libsystemd.so.0')
|
830
|
+
|
831
|
+
lib.sd_journal_sendv = lib['sd_journal_sendv'] # type: ignore
|
832
|
+
lib.sd_journal_sendv.restype = ct.c_int
|
833
|
+
lib.sd_journal_sendv.argtypes = [ct.POINTER(sd_iovec), ct.c_int]
|
834
|
+
|
835
|
+
return lib
|
836
|
+
|
837
|
+
|
838
|
+
@cached_nullary
|
839
|
+
def sd_try_libsystemd() -> ta.Optional[ta.Any]:
|
840
|
+
try:
|
841
|
+
return sd_libsystemd()
|
842
|
+
except OSError: # noqa
|
843
|
+
return None
|
844
|
+
|
845
|
+
|
846
|
+
##
|
847
|
+
|
848
|
+
|
849
|
+
def sd_journald_send(**fields: str) -> int:
|
850
|
+
lib = sd_libsystemd()
|
851
|
+
|
852
|
+
msgs = [
|
853
|
+
f'{k.upper()}={v}\0'.encode('utf-8')
|
854
|
+
for k, v in fields.items()
|
855
|
+
]
|
856
|
+
|
857
|
+
vec = (sd_iovec * len(msgs))()
|
858
|
+
cl = (ct.c_char_p * len(msgs))() # noqa
|
859
|
+
for i in range(len(msgs)):
|
860
|
+
vec[i].iov_base = ct.cast(ct.c_char_p(msgs[i]), ct.c_void_p)
|
861
|
+
vec[i].iov_len = len(msgs[i]) - 1
|
862
|
+
|
863
|
+
return lib.sd_journal_sendv(vec, len(msgs))
|
864
|
+
|
865
|
+
|
866
|
+
##
|
867
|
+
|
868
|
+
|
869
|
+
SD_LOG_LEVEL_MAP: ta.Mapping[int, int] = {
|
870
|
+
logging.FATAL: syslog.LOG_EMERG, # system is unusable
|
871
|
+
# LOG_ALERT ? # action must be taken immediately
|
872
|
+
logging.CRITICAL: syslog.LOG_CRIT,
|
873
|
+
logging.ERROR: syslog.LOG_ERR,
|
874
|
+
logging.WARNING: syslog.LOG_WARNING,
|
875
|
+
# LOG_NOTICE ? # normal but significant condition
|
876
|
+
logging.INFO: syslog.LOG_INFO,
|
877
|
+
logging.DEBUG: syslog.LOG_DEBUG,
|
878
|
+
}
|
879
|
+
|
880
|
+
|
881
|
+
class JournaldLogHandler(logging.Handler):
|
882
|
+
"""
|
883
|
+
TODO:
|
884
|
+
- fallback handler for when this barfs
|
885
|
+
"""
|
886
|
+
|
887
|
+
def __init__(
|
888
|
+
self,
|
889
|
+
*,
|
890
|
+
use_formatter_output: bool = False,
|
891
|
+
) -> None:
|
892
|
+
super().__init__()
|
893
|
+
|
894
|
+
sd_libsystemd()
|
895
|
+
|
896
|
+
self._use_formatter_output = use_formatter_output
|
897
|
+
|
898
|
+
#
|
899
|
+
|
900
|
+
EXTRA_RECORD_ATTRS_BY_JOURNALD_FIELD: ta.ClassVar[ta.Mapping[str, str]] = {
|
901
|
+
'name': 'name',
|
902
|
+
'module': 'module',
|
903
|
+
'exception': 'exc_text',
|
904
|
+
'thread_name': 'threadName',
|
905
|
+
'task_name': 'taskName',
|
906
|
+
}
|
907
|
+
|
908
|
+
def make_fields(self, record: logging.LogRecord) -> ta.Mapping[str, str]:
|
909
|
+
formatter_message = self.format(record)
|
910
|
+
if self._use_formatter_output:
|
911
|
+
message = formatter_message
|
912
|
+
else:
|
913
|
+
message = record.message
|
914
|
+
|
915
|
+
fields: dict[str, str] = {
|
916
|
+
'message': message,
|
917
|
+
'priority': str(SD_LOG_LEVEL_MAP[record.levelno]),
|
918
|
+
'tid': str(threading.get_ident()),
|
919
|
+
}
|
920
|
+
|
921
|
+
if (pathname := record.pathname) is not None:
|
922
|
+
fields['code_file'] = pathname
|
923
|
+
if (lineno := record.lineno) is not None:
|
924
|
+
fields['code_lineno'] = str(lineno)
|
925
|
+
if (func_name := record.funcName) is not None:
|
926
|
+
fields['code_func'] = func_name
|
927
|
+
|
928
|
+
for f, a in self.EXTRA_RECORD_ATTRS_BY_JOURNALD_FIELD.items():
|
929
|
+
if (v := getattr(record, a, None)) is not None:
|
930
|
+
fields[f] = str(v)
|
931
|
+
|
932
|
+
return fields
|
933
|
+
|
934
|
+
#
|
935
|
+
|
936
|
+
def emit(self, record: logging.LogRecord) -> None:
|
937
|
+
try:
|
938
|
+
fields = self.make_fields(record)
|
939
|
+
|
940
|
+
if rc := sd_journald_send(**fields):
|
941
|
+
raise RuntimeError(f'{self.__class__.__name__}.emit failed: {rc=}') # noqa
|
942
|
+
|
943
|
+
except RecursionError: # See issue 36272
|
944
|
+
raise
|
945
|
+
|
946
|
+
except Exception: # noqa
|
947
|
+
self.handleError(record)
|
948
|
+
|
949
|
+
|
950
|
+
def journald_log_handler_factory(
|
951
|
+
*,
|
952
|
+
no_tty_check: bool = False,
|
953
|
+
no_fallback: bool = False,
|
954
|
+
) -> logging.Handler:
|
955
|
+
if (
|
956
|
+
sys.platform == 'linux' and
|
957
|
+
(no_tty_check or not sys.stderr.isatty()) and
|
958
|
+
(no_fallback or sd_try_libsystemd() is not None)
|
959
|
+
):
|
960
|
+
return JournaldLogHandler()
|
961
|
+
|
962
|
+
return logging.StreamHandler()
|
963
|
+
|
964
|
+
|
1015
965
|
########################################
|
1016
966
|
# ../../../omlish/lite/logs.py
|
1017
967
|
"""
|
@@ -1099,7 +1049,7 @@ class StandardLogFormatter(logging.Formatter):
|
|
1099
1049
|
if datefmt:
|
1100
1050
|
return ct.strftime(datefmt) # noqa
|
1101
1051
|
else:
|
1102
|
-
t = ct.strftime(
|
1052
|
+
t = ct.strftime('%Y-%m-%d %H:%M:%S')
|
1103
1053
|
return '%s.%03d' % (t, record.msecs)
|
1104
1054
|
|
1105
1055
|
|
@@ -1236,6 +1186,7 @@ def configure_standard_logging(
|
|
1236
1186
|
json: bool = False,
|
1237
1187
|
target: ta.Optional[logging.Logger] = None,
|
1238
1188
|
force: bool = False,
|
1189
|
+
handler_factory: ta.Optional[ta.Callable[[], logging.Handler]] = None,
|
1239
1190
|
) -> ta.Optional[StandardLogHandler]:
|
1240
1191
|
with _locking_logging_module_lock():
|
1241
1192
|
if target is None:
|
@@ -1249,7 +1200,10 @@ def configure_standard_logging(
|
|
1249
1200
|
|
1250
1201
|
#
|
1251
1202
|
|
1252
|
-
|
1203
|
+
if handler_factory is not None:
|
1204
|
+
handler = handler_factory()
|
1205
|
+
else:
|
1206
|
+
handler = logging.StreamHandler()
|
1253
1207
|
|
1254
1208
|
#
|
1255
1209
|
|
@@ -1649,12 +1603,9 @@ class EventCallbacks:
|
|
1649
1603
|
|
1650
1604
|
EVENT_CALLBACKS = EventCallbacks()
|
1651
1605
|
|
1652
|
-
notify_event = EVENT_CALLBACKS.notify
|
1653
|
-
clear_events = EVENT_CALLBACKS.clear
|
1654
|
-
|
1655
1606
|
|
1656
1607
|
class Event(abc.ABC): # noqa
|
1657
|
-
"""Abstract event type
|
1608
|
+
"""Abstract event type."""
|
1658
1609
|
|
1659
1610
|
|
1660
1611
|
class ProcessLogEvent(Event, abc.ABC):
|
@@ -1734,7 +1685,7 @@ class RemoteCommunicationEvent(Event):
|
|
1734
1685
|
|
1735
1686
|
|
1736
1687
|
class SupervisorStateChangeEvent(Event):
|
1737
|
-
"""
|
1688
|
+
"""Abstract class."""
|
1738
1689
|
|
1739
1690
|
def payload(self):
|
1740
1691
|
return ''
|
@@ -1756,7 +1707,7 @@ class EventRejectedEvent: # purposely does not subclass Event
|
|
1756
1707
|
|
1757
1708
|
|
1758
1709
|
class ProcessStateEvent(Event):
|
1759
|
-
"""
|
1710
|
+
"""Abstract class, never raised directly."""
|
1760
1711
|
frm = None
|
1761
1712
|
to = None
|
1762
1713
|
|
@@ -1845,7 +1796,7 @@ class ProcessGroupRemovedEvent(ProcessGroupEvent):
|
|
1845
1796
|
|
1846
1797
|
|
1847
1798
|
class TickEvent(Event):
|
1848
|
-
"""
|
1799
|
+
"""Abstract."""
|
1849
1800
|
|
1850
1801
|
def __init__(self, when, supervisord):
|
1851
1802
|
super().__init__()
|
@@ -1888,39 +1839,273 @@ class EventTypes:
|
|
1888
1839
|
PROCESS_STATE_RUNNING = ProcessStateRunningEvent
|
1889
1840
|
PROCESS_STATE_UNKNOWN = ProcessStateUnknownEvent
|
1890
1841
|
|
1891
|
-
PROCESS_COMMUNICATION = ProcessCommunicationEvent # abstract
|
1892
|
-
PROCESS_COMMUNICATION_STDOUT = ProcessCommunicationStdoutEvent
|
1893
|
-
PROCESS_COMMUNICATION_STDERR = ProcessCommunicationStderrEvent
|
1842
|
+
PROCESS_COMMUNICATION = ProcessCommunicationEvent # abstract
|
1843
|
+
PROCESS_COMMUNICATION_STDOUT = ProcessCommunicationStdoutEvent
|
1844
|
+
PROCESS_COMMUNICATION_STDERR = ProcessCommunicationStderrEvent
|
1845
|
+
|
1846
|
+
PROCESS_LOG = ProcessLogEvent
|
1847
|
+
PROCESS_LOG_STDOUT = ProcessLogStdoutEvent
|
1848
|
+
PROCESS_LOG_STDERR = ProcessLogStderrEvent
|
1849
|
+
|
1850
|
+
REMOTE_COMMUNICATION = RemoteCommunicationEvent
|
1851
|
+
|
1852
|
+
SUPERVISOR_STATE_CHANGE = SupervisorStateChangeEvent # abstract
|
1853
|
+
SUPERVISOR_STATE_CHANGE_RUNNING = SupervisorRunningEvent
|
1854
|
+
SUPERVISOR_STATE_CHANGE_STOPPING = SupervisorStoppingEvent
|
1855
|
+
|
1856
|
+
TICK = TickEvent # abstract
|
1857
|
+
TICK_5 = Tick5Event
|
1858
|
+
TICK_60 = Tick60Event
|
1859
|
+
TICK_3600 = Tick3600Event
|
1860
|
+
|
1861
|
+
PROCESS_GROUP = ProcessGroupEvent # abstract
|
1862
|
+
PROCESS_GROUP_ADDED = ProcessGroupAddedEvent
|
1863
|
+
PROCESS_GROUP_REMOVED = ProcessGroupRemovedEvent
|
1864
|
+
|
1865
|
+
|
1866
|
+
def get_event_name_by_type(requested):
|
1867
|
+
for name, typ in EventTypes.__dict__.items():
|
1868
|
+
if typ is requested:
|
1869
|
+
return name
|
1870
|
+
return None
|
1871
|
+
|
1872
|
+
|
1873
|
+
def register(name, event):
|
1874
|
+
setattr(EventTypes, name, event)
|
1875
|
+
|
1876
|
+
|
1877
|
+
########################################
|
1878
|
+
# ../poller.py
|
1879
|
+
|
1880
|
+
|
1881
|
+
class BasePoller(abc.ABC):
|
1882
|
+
|
1883
|
+
def __init__(self) -> None:
|
1884
|
+
super().__init__()
|
1885
|
+
|
1886
|
+
@abc.abstractmethod
|
1887
|
+
def register_readable(self, fd: int) -> None:
|
1888
|
+
raise NotImplementedError
|
1889
|
+
|
1890
|
+
@abc.abstractmethod
|
1891
|
+
def register_writable(self, fd: int) -> None:
|
1892
|
+
raise NotImplementedError
|
1893
|
+
|
1894
|
+
@abc.abstractmethod
|
1895
|
+
def unregister_readable(self, fd: int) -> None:
|
1896
|
+
raise NotImplementedError
|
1897
|
+
|
1898
|
+
@abc.abstractmethod
|
1899
|
+
def unregister_writable(self, fd: int) -> None:
|
1900
|
+
raise NotImplementedError
|
1901
|
+
|
1902
|
+
@abc.abstractmethod
|
1903
|
+
def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[int], ta.List[int]]:
|
1904
|
+
raise NotImplementedError
|
1905
|
+
|
1906
|
+
def before_daemonize(self) -> None: # noqa
|
1907
|
+
pass
|
1908
|
+
|
1909
|
+
def after_daemonize(self) -> None: # noqa
|
1910
|
+
pass
|
1911
|
+
|
1912
|
+
def close(self) -> None: # noqa
|
1913
|
+
pass
|
1914
|
+
|
1915
|
+
|
1916
|
+
class SelectPoller(BasePoller):
|
1917
|
+
|
1918
|
+
def __init__(self) -> None:
|
1919
|
+
super().__init__()
|
1920
|
+
|
1921
|
+
self._readables: ta.Set[int] = set()
|
1922
|
+
self._writables: ta.Set[int] = set()
|
1923
|
+
|
1924
|
+
def register_readable(self, fd: int) -> None:
|
1925
|
+
self._readables.add(fd)
|
1926
|
+
|
1927
|
+
def register_writable(self, fd: int) -> None:
|
1928
|
+
self._writables.add(fd)
|
1929
|
+
|
1930
|
+
def unregister_readable(self, fd: int) -> None:
|
1931
|
+
self._readables.discard(fd)
|
1932
|
+
|
1933
|
+
def unregister_writable(self, fd: int) -> None:
|
1934
|
+
self._writables.discard(fd)
|
1935
|
+
|
1936
|
+
def unregister_all(self) -> None:
|
1937
|
+
self._readables.clear()
|
1938
|
+
self._writables.clear()
|
1939
|
+
|
1940
|
+
def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[int], ta.List[int]]:
|
1941
|
+
try:
|
1942
|
+
r, w, x = select.select(
|
1943
|
+
self._readables,
|
1944
|
+
self._writables,
|
1945
|
+
[], timeout,
|
1946
|
+
)
|
1947
|
+
except OSError as err:
|
1948
|
+
if err.args[0] == errno.EINTR:
|
1949
|
+
log.debug('EINTR encountered in poll')
|
1950
|
+
return [], []
|
1951
|
+
if err.args[0] == errno.EBADF:
|
1952
|
+
log.debug('EBADF encountered in poll')
|
1953
|
+
self.unregister_all()
|
1954
|
+
return [], []
|
1955
|
+
raise
|
1956
|
+
return r, w
|
1957
|
+
|
1958
|
+
|
1959
|
+
class PollPoller(BasePoller):
|
1960
|
+
_READ = select.POLLIN | select.POLLPRI | select.POLLHUP
|
1961
|
+
_WRITE = select.POLLOUT
|
1962
|
+
|
1963
|
+
def __init__(self) -> None:
|
1964
|
+
super().__init__()
|
1965
|
+
|
1966
|
+
self._poller = select.poll()
|
1967
|
+
self._readables: set[int] = set()
|
1968
|
+
self._writables: set[int] = set()
|
1969
|
+
|
1970
|
+
def register_readable(self, fd: int) -> None:
|
1971
|
+
self._poller.register(fd, self._READ)
|
1972
|
+
self._readables.add(fd)
|
1973
|
+
|
1974
|
+
def register_writable(self, fd: int) -> None:
|
1975
|
+
self._poller.register(fd, self._WRITE)
|
1976
|
+
self._writables.add(fd)
|
1977
|
+
|
1978
|
+
def unregister_readable(self, fd: int) -> None:
|
1979
|
+
self._readables.discard(fd)
|
1980
|
+
self._poller.unregister(fd)
|
1981
|
+
if fd in self._writables:
|
1982
|
+
self._poller.register(fd, self._WRITE)
|
1983
|
+
|
1984
|
+
def unregister_writable(self, fd: int) -> None:
|
1985
|
+
self._writables.discard(fd)
|
1986
|
+
self._poller.unregister(fd)
|
1987
|
+
if fd in self._readables:
|
1988
|
+
self._poller.register(fd, self._READ)
|
1989
|
+
|
1990
|
+
def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[int], ta.List[int]]:
|
1991
|
+
fds = self._poll_fds(timeout) # type: ignore
|
1992
|
+
readables, writables = [], []
|
1993
|
+
for fd, eventmask in fds:
|
1994
|
+
if self._ignore_invalid(fd, eventmask):
|
1995
|
+
continue
|
1996
|
+
if eventmask & self._READ:
|
1997
|
+
readables.append(fd)
|
1998
|
+
if eventmask & self._WRITE:
|
1999
|
+
writables.append(fd)
|
2000
|
+
return readables, writables
|
2001
|
+
|
2002
|
+
def _poll_fds(self, timeout: float) -> ta.List[ta.Tuple[int, int]]:
|
2003
|
+
try:
|
2004
|
+
return self._poller.poll(timeout * 1000)
|
2005
|
+
except OSError as err:
|
2006
|
+
if err.args[0] == errno.EINTR:
|
2007
|
+
log.debug('EINTR encountered in poll')
|
2008
|
+
return []
|
2009
|
+
raise
|
2010
|
+
|
2011
|
+
def _ignore_invalid(self, fd: int, eventmask: int) -> bool:
|
2012
|
+
if eventmask & select.POLLNVAL:
|
2013
|
+
# POLLNVAL means `fd` value is invalid, not open. When a process quits it's `fd`s are closed so there is no
|
2014
|
+
# more reason to keep this `fd` registered If the process restarts it's `fd`s are registered again.
|
2015
|
+
self._poller.unregister(fd)
|
2016
|
+
self._readables.discard(fd)
|
2017
|
+
self._writables.discard(fd)
|
2018
|
+
return True
|
2019
|
+
return False
|
2020
|
+
|
2021
|
+
|
2022
|
+
if sys.platform == 'darwin' or sys.platform.startswith('freebsd'):
|
2023
|
+
class KqueuePoller(BasePoller):
|
2024
|
+
max_events = 1000
|
2025
|
+
|
2026
|
+
def __init__(self) -> None:
|
2027
|
+
super().__init__()
|
2028
|
+
|
2029
|
+
self._kqueue: ta.Optional[ta.Any] = select.kqueue()
|
2030
|
+
self._readables: set[int] = set()
|
2031
|
+
self._writables: set[int] = set()
|
2032
|
+
|
2033
|
+
def register_readable(self, fd: int) -> None:
|
2034
|
+
self._readables.add(fd)
|
2035
|
+
kevent = select.kevent(fd, filter=select.KQ_FILTER_READ, flags=select.KQ_EV_ADD)
|
2036
|
+
self._kqueue_control(fd, kevent)
|
2037
|
+
|
2038
|
+
def register_writable(self, fd: int) -> None:
|
2039
|
+
self._writables.add(fd)
|
2040
|
+
kevent = select.kevent(fd, filter=select.KQ_FILTER_WRITE, flags=select.KQ_EV_ADD)
|
2041
|
+
self._kqueue_control(fd, kevent)
|
2042
|
+
|
2043
|
+
def unregister_readable(self, fd: int) -> None:
|
2044
|
+
kevent = select.kevent(fd, filter=select.KQ_FILTER_READ, flags=select.KQ_EV_DELETE)
|
2045
|
+
self._readables.discard(fd)
|
2046
|
+
self._kqueue_control(fd, kevent)
|
2047
|
+
|
2048
|
+
def unregister_writable(self, fd: int) -> None:
|
2049
|
+
kevent = select.kevent(fd, filter=select.KQ_FILTER_WRITE, flags=select.KQ_EV_DELETE)
|
2050
|
+
self._writables.discard(fd)
|
2051
|
+
self._kqueue_control(fd, kevent)
|
2052
|
+
|
2053
|
+
def _kqueue_control(self, fd: int, kevent: 'select.kevent') -> None:
|
2054
|
+
try:
|
2055
|
+
self._kqueue.control([kevent], 0) # type: ignore
|
2056
|
+
except OSError as error:
|
2057
|
+
if error.errno == errno.EBADF:
|
2058
|
+
log.debug('EBADF encountered in kqueue. Invalid file descriptor %s', fd)
|
2059
|
+
else:
|
2060
|
+
raise
|
2061
|
+
|
2062
|
+
def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[int], ta.List[int]]:
|
2063
|
+
readables, writables = [], [] # type: ignore
|
1894
2064
|
|
1895
|
-
|
1896
|
-
|
1897
|
-
|
2065
|
+
try:
|
2066
|
+
kevents = self._kqueue.control(None, self.max_events, timeout) # type: ignore
|
2067
|
+
except OSError as error:
|
2068
|
+
if error.errno == errno.EINTR:
|
2069
|
+
log.debug('EINTR encountered in poll')
|
2070
|
+
return readables, writables
|
2071
|
+
raise
|
1898
2072
|
|
1899
|
-
|
2073
|
+
for kevent in kevents:
|
2074
|
+
if kevent.filter == select.KQ_FILTER_READ:
|
2075
|
+
readables.append(kevent.ident)
|
2076
|
+
if kevent.filter == select.KQ_FILTER_WRITE:
|
2077
|
+
writables.append(kevent.ident)
|
1900
2078
|
|
1901
|
-
|
1902
|
-
SUPERVISOR_STATE_CHANGE_RUNNING = SupervisorRunningEvent
|
1903
|
-
SUPERVISOR_STATE_CHANGE_STOPPING = SupervisorStoppingEvent
|
2079
|
+
return readables, writables
|
1904
2080
|
|
1905
|
-
|
1906
|
-
|
1907
|
-
TICK_60 = Tick60Event
|
1908
|
-
TICK_3600 = Tick3600Event
|
2081
|
+
def before_daemonize(self) -> None:
|
2082
|
+
self.close()
|
1909
2083
|
|
1910
|
-
|
1911
|
-
|
1912
|
-
|
2084
|
+
def after_daemonize(self) -> None:
|
2085
|
+
self._kqueue = select.kqueue()
|
2086
|
+
for fd in self._readables:
|
2087
|
+
self.register_readable(fd)
|
2088
|
+
for fd in self._writables:
|
2089
|
+
self.register_writable(fd)
|
1913
2090
|
|
2091
|
+
def close(self) -> None:
|
2092
|
+
self._kqueue.close() # type: ignore
|
2093
|
+
self._kqueue = None
|
1914
2094
|
|
1915
|
-
|
1916
|
-
|
1917
|
-
if typ is requested:
|
1918
|
-
return name
|
1919
|
-
return None
|
2095
|
+
else:
|
2096
|
+
KqueuePoller = None
|
1920
2097
|
|
1921
2098
|
|
1922
|
-
|
1923
|
-
|
2099
|
+
Poller: ta.Type[BasePoller]
|
2100
|
+
if (
|
2101
|
+
sys.platform == 'darwin' or sys.platform.startswith('freebsd') and
|
2102
|
+
hasattr(select, 'kqueue') and KqueuePoller is not None
|
2103
|
+
):
|
2104
|
+
Poller = KqueuePoller
|
2105
|
+
elif hasattr(select, 'poll'):
|
2106
|
+
Poller = PollPoller
|
2107
|
+
else:
|
2108
|
+
Poller = SelectPoller
|
1924
2109
|
|
1925
2110
|
|
1926
2111
|
########################################
|
@@ -1973,41 +2158,47 @@ class AbstractSubprocess(abc.ABC):
|
|
1973
2158
|
# ../context.py
|
1974
2159
|
|
1975
2160
|
|
1976
|
-
log = logging.getLogger(__name__)
|
1977
|
-
|
1978
|
-
|
1979
2161
|
class ServerContext(AbstractServerContext):
|
1980
|
-
|
1981
|
-
|
1982
|
-
|
1983
|
-
|
1984
|
-
|
1985
|
-
|
2162
|
+
def __init__(
|
2163
|
+
self,
|
2164
|
+
config: ServerConfig,
|
2165
|
+
*,
|
2166
|
+
epoch: int = 0,
|
2167
|
+
) -> None:
|
1986
2168
|
super().__init__()
|
1987
2169
|
|
1988
2170
|
self._config = config
|
2171
|
+
self._epoch = epoch
|
1989
2172
|
|
1990
2173
|
self._pid_history: ta.Dict[int, AbstractSubprocess] = {}
|
1991
2174
|
self._state: SupervisorState = SupervisorStates.RUNNING
|
1992
2175
|
|
1993
|
-
self.
|
2176
|
+
self._signal_receiver = SignalReceiver()
|
1994
2177
|
|
1995
|
-
self.
|
2178
|
+
self._poller: BasePoller = Poller()
|
1996
2179
|
|
1997
|
-
if
|
1998
|
-
uid = name_to_uid(
|
1999
|
-
self.
|
2000
|
-
self.
|
2180
|
+
if config.user is not None:
|
2181
|
+
uid = name_to_uid(config.user)
|
2182
|
+
self._uid: ta.Optional[int] = uid
|
2183
|
+
self._gid: ta.Optional[int] = gid_for_uid(uid)
|
2001
2184
|
else:
|
2002
|
-
self.
|
2003
|
-
self.
|
2185
|
+
self._uid = None
|
2186
|
+
self._gid = None
|
2004
2187
|
|
2005
|
-
self.
|
2188
|
+
self._unlink_pidfile = False
|
2006
2189
|
|
2007
2190
|
@property
|
2008
2191
|
def config(self) -> ServerConfig:
|
2009
2192
|
return self._config
|
2010
2193
|
|
2194
|
+
@property
|
2195
|
+
def epoch(self) -> int:
|
2196
|
+
return self._epoch
|
2197
|
+
|
2198
|
+
@property
|
2199
|
+
def first(self) -> bool:
|
2200
|
+
return not self._epoch
|
2201
|
+
|
2011
2202
|
@property
|
2012
2203
|
def state(self) -> SupervisorState:
|
2013
2204
|
return self._state
|
@@ -2015,17 +2206,26 @@ class ServerContext(AbstractServerContext):
|
|
2015
2206
|
def set_state(self, state: SupervisorState) -> None:
|
2016
2207
|
self._state = state
|
2017
2208
|
|
2209
|
+
@property
|
2210
|
+
def poller(self) -> BasePoller:
|
2211
|
+
return self._poller
|
2212
|
+
|
2018
2213
|
@property
|
2019
2214
|
def pid_history(self) -> ta.Dict[int, AbstractSubprocess]:
|
2020
2215
|
return self._pid_history
|
2021
2216
|
|
2022
|
-
|
2023
|
-
|
2217
|
+
@property
|
2218
|
+
def uid(self) -> ta.Optional[int]:
|
2219
|
+
return self._uid
|
2220
|
+
|
2221
|
+
@property
|
2222
|
+
def gid(self) -> ta.Optional[int]:
|
2223
|
+
return self._gid
|
2024
2224
|
|
2025
2225
|
##
|
2026
2226
|
|
2027
2227
|
def set_signals(self) -> None:
|
2028
|
-
self.
|
2228
|
+
self._signal_receiver.install(
|
2029
2229
|
signal.SIGTERM,
|
2030
2230
|
signal.SIGINT,
|
2031
2231
|
signal.SIGQUIT,
|
@@ -2136,7 +2336,7 @@ class ServerContext(AbstractServerContext):
|
|
2136
2336
|
))
|
2137
2337
|
|
2138
2338
|
def cleanup(self) -> None:
|
2139
|
-
if self.
|
2339
|
+
if self._unlink_pidfile:
|
2140
2340
|
try_unlink(self.config.pidfile)
|
2141
2341
|
self.poller.close()
|
2142
2342
|
|
@@ -2189,6 +2389,7 @@ class ServerContext(AbstractServerContext):
|
|
2189
2389
|
# Parent
|
2190
2390
|
log.debug('supervisord forked; parent exiting')
|
2191
2391
|
real_exit(0)
|
2392
|
+
|
2192
2393
|
# Child
|
2193
2394
|
log.info('daemonizing the supervisord process')
|
2194
2395
|
if self.config.directory:
|
@@ -2198,11 +2399,15 @@ class ServerContext(AbstractServerContext):
|
|
2198
2399
|
log.critical("can't chdir into %r: %s", self.config.directory, err)
|
2199
2400
|
else:
|
2200
2401
|
log.info('set current directory: %r', self.config.directory)
|
2402
|
+
|
2201
2403
|
os.dup2(0, os.open('/dev/null', os.O_RDONLY))
|
2202
2404
|
os.dup2(1, os.open('/dev/null', os.O_WRONLY))
|
2203
2405
|
os.dup2(2, os.open('/dev/null', os.O_WRONLY))
|
2406
|
+
|
2204
2407
|
os.setsid()
|
2408
|
+
|
2205
2409
|
os.umask(self.config.umask)
|
2410
|
+
|
2206
2411
|
# XXX Stevens, in his Advanced Unix book, section 13.3 (page 417) recommends calling umask(0) and closing unused
|
2207
2412
|
# file descriptors. In his Network Programming book, he additionally recommends ignoring SIGHUP and forking
|
2208
2413
|
# again after the setsid() call, for obscure SVR4 reasons.
|
@@ -2217,7 +2422,7 @@ class ServerContext(AbstractServerContext):
|
|
2217
2422
|
return logfile
|
2218
2423
|
|
2219
2424
|
def get_signal(self) -> ta.Optional[int]:
|
2220
|
-
return self.
|
2425
|
+
return self._signal_receiver.get_signal()
|
2221
2426
|
|
2222
2427
|
def write_pidfile(self) -> None:
|
2223
2428
|
pid = os.getpid()
|
@@ -2227,7 +2432,7 @@ class ServerContext(AbstractServerContext):
|
|
2227
2432
|
except OSError:
|
2228
2433
|
log.critical('could not write pidfile %s', self.config.pidfile)
|
2229
2434
|
else:
|
2230
|
-
self.
|
2435
|
+
self._unlink_pidfile = True
|
2231
2436
|
log.info('supervisord started with pid %s', pid)
|
2232
2437
|
|
2233
2438
|
|
@@ -2280,11 +2485,14 @@ def drop_privileges(user: ta.Union[int, str, None]) -> ta.Optional[str]:
|
|
2280
2485
|
os.setgroups(groups)
|
2281
2486
|
except OSError:
|
2282
2487
|
return 'Could not set groups of effective user'
|
2488
|
+
|
2283
2489
|
try:
|
2284
2490
|
os.setgid(gid)
|
2285
2491
|
except OSError:
|
2286
2492
|
return 'Could not set group id of effective user'
|
2493
|
+
|
2287
2494
|
os.setuid(uid)
|
2495
|
+
|
2288
2496
|
return None
|
2289
2497
|
|
2290
2498
|
|
@@ -2354,9 +2562,6 @@ def check_execv_args(filename, argv, st) -> None:
|
|
2354
2562
|
# ../dispatchers.py
|
2355
2563
|
|
2356
2564
|
|
2357
|
-
log = logging.getLogger(__name__)
|
2358
|
-
|
2359
|
-
|
2360
2565
|
class Dispatcher(abc.ABC):
|
2361
2566
|
|
2362
2567
|
def __init__(self, process: AbstractSubprocess, channel: str, fd: int) -> None:
|
@@ -2513,16 +2718,16 @@ class OutputDispatcher(Dispatcher):
|
|
2513
2718
|
# )
|
2514
2719
|
|
2515
2720
|
def remove_logs(self):
|
2516
|
-
for
|
2517
|
-
if
|
2518
|
-
for handler in
|
2721
|
+
for l in (self._normal_log, self._capture_log):
|
2722
|
+
if l is not None:
|
2723
|
+
for handler in l.handlers:
|
2519
2724
|
handler.remove() # type: ignore
|
2520
2725
|
handler.reopen() # type: ignore
|
2521
2726
|
|
2522
2727
|
def reopen_logs(self):
|
2523
|
-
for
|
2524
|
-
if
|
2525
|
-
for handler in
|
2728
|
+
for l in (self._normal_log, self._capture_log):
|
2729
|
+
if l is not None:
|
2730
|
+
for handler in l.handlers:
|
2526
2731
|
handler.reopen() # type: ignore
|
2527
2732
|
|
2528
2733
|
def _log(self, data):
|
@@ -2545,10 +2750,10 @@ class OutputDispatcher(Dispatcher):
|
|
2545
2750
|
|
2546
2751
|
if self._channel == 'stdout':
|
2547
2752
|
if self._stdout_events_enabled:
|
2548
|
-
|
2753
|
+
EVENT_CALLBACKS.notify(ProcessLogStdoutEvent(self._process, self._process.pid, data))
|
2549
2754
|
|
2550
2755
|
elif self._stderr_events_enabled:
|
2551
|
-
|
2756
|
+
EVENT_CALLBACKS.notify(ProcessLogStderrEvent(self._process, self._process.pid, data))
|
2552
2757
|
|
2553
2758
|
def record_output(self):
|
2554
2759
|
if self._capture_log is None:
|
@@ -2599,7 +2804,7 @@ class OutputDispatcher(Dispatcher):
|
|
2599
2804
|
channel = self._channel
|
2600
2805
|
procname = self._process.config.name
|
2601
2806
|
event = self.event_type(self._process, self._process.pid, data)
|
2602
|
-
|
2807
|
+
EVENT_CALLBACKS.notify(event)
|
2603
2808
|
|
2604
2809
|
log.debug('%r %s emitted a comm event', procname, channel)
|
2605
2810
|
for handler in self._capture_log.handlers:
|
@@ -2660,9 +2865,6 @@ class InputDispatcher(Dispatcher):
|
|
2660
2865
|
# ../process.py
|
2661
2866
|
|
2662
2867
|
|
2663
|
-
log = logging.getLogger(__name__)
|
2664
|
-
|
2665
|
-
|
2666
2868
|
@functools.total_ordering
|
2667
2869
|
class Subprocess(AbstractSubprocess):
|
2668
2870
|
"""A class to manage a subprocess."""
|
@@ -2815,7 +3017,7 @@ class Subprocess(AbstractSubprocess):
|
|
2815
3017
|
event_class = self.event_map.get(new_state)
|
2816
3018
|
if event_class is not None:
|
2817
3019
|
event = event_class(self, old_state, expected)
|
2818
|
-
|
3020
|
+
EVENT_CALLBACKS.notify(event)
|
2819
3021
|
|
2820
3022
|
return True
|
2821
3023
|
|
@@ -2932,6 +3134,7 @@ class Subprocess(AbstractSubprocess):
|
|
2932
3134
|
os.dup2(self._pipes['child_stdout'], 2)
|
2933
3135
|
else:
|
2934
3136
|
os.dup2(self._pipes['child_stderr'], 2)
|
3137
|
+
# FIXME: leave debugger fds
|
2935
3138
|
for i in range(3, self.context.config.minfds):
|
2936
3139
|
close_fd(i)
|
2937
3140
|
|
@@ -2967,7 +3170,7 @@ class Subprocess(AbstractSubprocess):
|
|
2967
3170
|
cwd = self.config.directory
|
2968
3171
|
try:
|
2969
3172
|
if cwd is not None:
|
2970
|
-
os.chdir(cwd)
|
3173
|
+
os.chdir(os.path.expanduser(cwd))
|
2971
3174
|
except OSError as why:
|
2972
3175
|
code = errno.errorcode.get(why.args[0], why.args[0])
|
2973
3176
|
msg = f"couldn't chdir to {cwd}: {code}\n"
|
@@ -3023,7 +3226,7 @@ class Subprocess(AbstractSubprocess):
|
|
3023
3226
|
return self.kill(self.config.stopsignal)
|
3024
3227
|
|
3025
3228
|
def stop_report(self) -> None:
|
3026
|
-
"""
|
3229
|
+
"""Log a 'waiting for x to stop' message with throttling."""
|
3027
3230
|
if self.state == ProcessStates.STOPPING:
|
3028
3231
|
now = time.time()
|
3029
3232
|
|
@@ -3153,7 +3356,7 @@ class Subprocess(AbstractSubprocess):
|
|
3153
3356
|
return None
|
3154
3357
|
|
3155
3358
|
def finish(self, sts: int) -> None:
|
3156
|
-
"""
|
3359
|
+
"""The process was reaped and we need to report and manage its state."""
|
3157
3360
|
|
3158
3361
|
self.drain()
|
3159
3362
|
|
@@ -3234,7 +3437,7 @@ class Subprocess(AbstractSubprocess):
|
|
3234
3437
|
# system that this event was rejected so it can be processed again.
|
3235
3438
|
if self.event is not None:
|
3236
3439
|
# Note: this should only be true if we were in the BUSY state when finish() was called.
|
3237
|
-
|
3440
|
+
EVENT_CALLBACKS.notify(EventRejectedEvent(self, self.event)) # type: ignore
|
3238
3441
|
self.event = None
|
3239
3442
|
|
3240
3443
|
def set_uid(self) -> ta.Optional[str]:
|
@@ -3397,10 +3600,11 @@ class ProcessGroup:
|
|
3397
3600
|
|
3398
3601
|
|
3399
3602
|
########################################
|
3400
|
-
# supervisor.py
|
3603
|
+
# ../supervisor.py
|
3401
3604
|
|
3402
3605
|
|
3403
|
-
|
3606
|
+
def timeslice(period: int, when: float) -> int:
|
3607
|
+
return int(when - (when % period))
|
3404
3608
|
|
3405
3609
|
|
3406
3610
|
class Supervisor:
|
@@ -3415,6 +3619,8 @@ class Supervisor:
|
|
3415
3619
|
self._stopping = False # set after we detect that we are handling a stop request
|
3416
3620
|
self._last_shutdown_report = 0. # throttle for delayed process error reports at stop
|
3417
3621
|
|
3622
|
+
#
|
3623
|
+
|
3418
3624
|
@property
|
3419
3625
|
def context(self) -> ServerContext:
|
3420
3626
|
return self._context
|
@@ -3422,47 +3628,14 @@ class Supervisor:
|
|
3422
3628
|
def get_state(self) -> SupervisorState:
|
3423
3629
|
return self._context.state
|
3424
3630
|
|
3425
|
-
|
3426
|
-
if not self._context.first:
|
3427
|
-
# prevent crash on libdispatch-based systems, at least for the first request
|
3428
|
-
self._context.cleanup_fds()
|
3429
|
-
|
3430
|
-
self._context.set_uid_or_exit()
|
3431
|
-
|
3432
|
-
if self._context.first:
|
3433
|
-
self._context.set_rlimits_or_exit()
|
3434
|
-
|
3435
|
-
# this sets the options.logger object delay logger instantiation until after setuid
|
3436
|
-
if not self._context.config.nocleanup:
|
3437
|
-
# clean up old automatic logs
|
3438
|
-
self._context.clear_auto_child_logdir()
|
3439
|
-
|
3440
|
-
self.run()
|
3441
|
-
|
3442
|
-
def run(self) -> None:
|
3443
|
-
self._process_groups = {} # clear
|
3444
|
-
self._stop_groups = None # clear
|
3445
|
-
|
3446
|
-
clear_events()
|
3447
|
-
|
3448
|
-
try:
|
3449
|
-
for config in self._context.config.groups or []:
|
3450
|
-
self.add_process_group(config)
|
3451
|
-
|
3452
|
-
self._context.set_signals()
|
3453
|
-
|
3454
|
-
if not self._context.config.nodaemon and self._context.first:
|
3455
|
-
self._context.daemonize()
|
3456
|
-
|
3457
|
-
# writing pid file needs to come *after* daemonizing or pid will be wrong
|
3458
|
-
self._context.write_pidfile()
|
3459
|
-
|
3460
|
-
self.runforever()
|
3631
|
+
#
|
3461
3632
|
|
3462
|
-
|
3463
|
-
|
3633
|
+
class DiffToActive(ta.NamedTuple):
|
3634
|
+
added: ta.List[ProcessGroupConfig]
|
3635
|
+
changed: ta.List[ProcessGroupConfig]
|
3636
|
+
removed: ta.List[ProcessGroupConfig]
|
3464
3637
|
|
3465
|
-
def diff_to_active(self):
|
3638
|
+
def diff_to_active(self) -> DiffToActive:
|
3466
3639
|
new = self._context.config.groups or []
|
3467
3640
|
cur = [group.config for group in self._process_groups.values()]
|
3468
3641
|
|
@@ -3474,7 +3647,7 @@ class Supervisor:
|
|
3474
3647
|
|
3475
3648
|
changed = [cand for cand in new if cand != curdict.get(cand.name, cand)]
|
3476
3649
|
|
3477
|
-
return added, changed, removed
|
3650
|
+
return Supervisor.DiffToActive(added, changed, removed)
|
3478
3651
|
|
3479
3652
|
def add_process_group(self, config: ProcessGroupConfig) -> bool:
|
3480
3653
|
name = config.name
|
@@ -3484,7 +3657,7 @@ class Supervisor:
|
|
3484
3657
|
group = self._process_groups[name] = ProcessGroup(config, self._context)
|
3485
3658
|
group.after_setuid()
|
3486
3659
|
|
3487
|
-
|
3660
|
+
EVENT_CALLBACKS.notify(ProcessGroupAddedEvent(name))
|
3488
3661
|
return True
|
3489
3662
|
|
3490
3663
|
def remove_process_group(self, name: str) -> bool:
|
@@ -3495,7 +3668,7 @@ class Supervisor:
|
|
3495
3668
|
|
3496
3669
|
del self._process_groups[name]
|
3497
3670
|
|
3498
|
-
|
3671
|
+
EVENT_CALLBACKS.notify(ProcessGroupRemovedEvent(name))
|
3499
3672
|
return True
|
3500
3673
|
|
3501
3674
|
def get_process_map(self) -> ta.Dict[int, Dispatcher]:
|
@@ -3524,6 +3697,72 @@ class Supervisor:
|
|
3524
3697
|
|
3525
3698
|
return unstopped
|
3526
3699
|
|
3700
|
+
#
|
3701
|
+
|
3702
|
+
def main(self) -> None:
|
3703
|
+
self.setup()
|
3704
|
+
self.run()
|
3705
|
+
|
3706
|
+
@cached_nullary
|
3707
|
+
def setup(self) -> None:
|
3708
|
+
if not self._context.first:
|
3709
|
+
# prevent crash on libdispatch-based systems, at least for the first request
|
3710
|
+
self._context.cleanup_fds()
|
3711
|
+
|
3712
|
+
self._context.set_uid_or_exit()
|
3713
|
+
|
3714
|
+
if self._context.first:
|
3715
|
+
self._context.set_rlimits_or_exit()
|
3716
|
+
|
3717
|
+
# this sets the options.logger object delay logger instantiation until after setuid
|
3718
|
+
if not self._context.config.nocleanup:
|
3719
|
+
# clean up old automatic logs
|
3720
|
+
self._context.clear_auto_child_logdir()
|
3721
|
+
|
3722
|
+
def run(
|
3723
|
+
self,
|
3724
|
+
*,
|
3725
|
+
callback: ta.Optional[ta.Callable[['Supervisor'], bool]] = None,
|
3726
|
+
) -> None:
|
3727
|
+
self._process_groups = {} # clear
|
3728
|
+
self._stop_groups = None # clear
|
3729
|
+
|
3730
|
+
EVENT_CALLBACKS.clear()
|
3731
|
+
|
3732
|
+
try:
|
3733
|
+
for config in self._context.config.groups or []:
|
3734
|
+
self.add_process_group(config)
|
3735
|
+
|
3736
|
+
self._context.set_signals()
|
3737
|
+
|
3738
|
+
if not self._context.config.nodaemon and self._context.first:
|
3739
|
+
self._context.daemonize()
|
3740
|
+
|
3741
|
+
# writing pid file needs to come *after* daemonizing or pid will be wrong
|
3742
|
+
self._context.write_pidfile()
|
3743
|
+
|
3744
|
+
EVENT_CALLBACKS.notify(SupervisorRunningEvent())
|
3745
|
+
|
3746
|
+
while True:
|
3747
|
+
if callback is not None and not callback(self):
|
3748
|
+
break
|
3749
|
+
|
3750
|
+
self._run_once()
|
3751
|
+
|
3752
|
+
finally:
|
3753
|
+
self._context.cleanup()
|
3754
|
+
|
3755
|
+
#
|
3756
|
+
|
3757
|
+
def _run_once(self) -> None:
|
3758
|
+
self._poll()
|
3759
|
+
self._reap()
|
3760
|
+
self._handle_signal()
|
3761
|
+
self._tick()
|
3762
|
+
|
3763
|
+
if self._context.state < SupervisorStates.RUNNING:
|
3764
|
+
self._ordered_stop_groups_phase_2()
|
3765
|
+
|
3527
3766
|
def _ordered_stop_groups_phase_1(self) -> None:
|
3528
3767
|
if self._stop_groups:
|
3529
3768
|
# stop the last group (the one with the "highest" priority)
|
@@ -3540,110 +3779,77 @@ class Supervisor:
|
|
3540
3779
|
# down, so push it back on to the end of the stop group queue
|
3541
3780
|
self._stop_groups.append(group)
|
3542
3781
|
|
3543
|
-
def
|
3544
|
-
|
3545
|
-
|
3546
|
-
|
3547
|
-
while True:
|
3548
|
-
combined_map = {}
|
3549
|
-
combined_map.update(self.get_process_map())
|
3782
|
+
def _poll(self) -> None:
|
3783
|
+
combined_map = {}
|
3784
|
+
combined_map.update(self.get_process_map())
|
3550
3785
|
|
3551
|
-
|
3552
|
-
|
3786
|
+
pgroups = list(self._process_groups.values())
|
3787
|
+
pgroups.sort()
|
3553
3788
|
|
3554
|
-
|
3555
|
-
|
3556
|
-
|
3557
|
-
|
3558
|
-
|
3559
|
-
|
3789
|
+
if self._context.state < SupervisorStates.RUNNING:
|
3790
|
+
if not self._stopping:
|
3791
|
+
# first time, set the stopping flag, do a notification and set stop_groups
|
3792
|
+
self._stopping = True
|
3793
|
+
self._stop_groups = pgroups[:]
|
3794
|
+
EVENT_CALLBACKS.notify(SupervisorStoppingEvent())
|
3560
3795
|
|
3561
|
-
|
3796
|
+
self._ordered_stop_groups_phase_1()
|
3562
3797
|
|
3563
|
-
|
3564
|
-
|
3565
|
-
|
3798
|
+
if not self.shutdown_report():
|
3799
|
+
# if there are no unstopped processes (we're done killing everything), it's OK to shutdown or reload
|
3800
|
+
raise ExitNow
|
3566
3801
|
|
3567
|
-
|
3568
|
-
|
3569
|
-
|
3570
|
-
|
3571
|
-
|
3802
|
+
for fd, dispatcher in combined_map.items():
|
3803
|
+
if dispatcher.readable():
|
3804
|
+
self._context.poller.register_readable(fd)
|
3805
|
+
if dispatcher.writable():
|
3806
|
+
self._context.poller.register_writable(fd)
|
3572
3807
|
|
3573
|
-
|
3808
|
+
timeout = 1 # this cannot be fewer than the smallest TickEvent (5)
|
3809
|
+
r, w = self._context.poller.poll(timeout)
|
3574
3810
|
|
3575
|
-
|
3576
|
-
|
3577
|
-
|
3578
|
-
|
3579
|
-
|
3580
|
-
|
3581
|
-
|
3582
|
-
self._context.poller.unregister_readable(fd)
|
3583
|
-
except ExitNow:
|
3584
|
-
raise
|
3585
|
-
except Exception: # noqa
|
3586
|
-
combined_map[fd].handle_error()
|
3587
|
-
else:
|
3588
|
-
# if the fd is not in combined_map, we should unregister it. otherwise, it will be polled every
|
3589
|
-
# time, which may cause 100% cpu usage
|
3590
|
-
log.debug('unexpected read event from fd %r', fd)
|
3591
|
-
try:
|
3811
|
+
for fd in r:
|
3812
|
+
if fd in combined_map:
|
3813
|
+
try:
|
3814
|
+
dispatcher = combined_map[fd]
|
3815
|
+
log.debug('read event caused by %r', dispatcher)
|
3816
|
+
dispatcher.handle_read_event()
|
3817
|
+
if not dispatcher.readable():
|
3592
3818
|
self._context.poller.unregister_readable(fd)
|
3593
|
-
|
3594
|
-
|
3819
|
+
except ExitNow:
|
3820
|
+
raise
|
3821
|
+
except Exception: # noqa
|
3822
|
+
combined_map[fd].handle_error()
|
3823
|
+
else:
|
3824
|
+
# if the fd is not in combined_map, we should unregister it. otherwise, it will be polled every
|
3825
|
+
# time, which may cause 100% cpu usage
|
3826
|
+
log.debug('unexpected read event from fd %r', fd)
|
3827
|
+
try:
|
3828
|
+
self._context.poller.unregister_readable(fd)
|
3829
|
+
except Exception: # noqa
|
3830
|
+
pass
|
3595
3831
|
|
3596
|
-
|
3597
|
-
|
3598
|
-
|
3599
|
-
|
3600
|
-
|
3601
|
-
|
3602
|
-
|
3603
|
-
self._context.poller.unregister_writable(fd)
|
3604
|
-
except ExitNow:
|
3605
|
-
raise
|
3606
|
-
except Exception: # noqa
|
3607
|
-
combined_map[fd].handle_error()
|
3608
|
-
else:
|
3609
|
-
log.debug('unexpected write event from fd %r', fd)
|
3610
|
-
try:
|
3832
|
+
for fd in w:
|
3833
|
+
if fd in combined_map:
|
3834
|
+
try:
|
3835
|
+
dispatcher = combined_map[fd]
|
3836
|
+
log.debug('write event caused by %r', dispatcher)
|
3837
|
+
dispatcher.handle_write_event()
|
3838
|
+
if not dispatcher.writable():
|
3611
3839
|
self._context.poller.unregister_writable(fd)
|
3612
|
-
|
3613
|
-
|
3614
|
-
|
3615
|
-
|
3616
|
-
|
3617
|
-
|
3618
|
-
|
3619
|
-
|
3620
|
-
|
3621
|
-
|
3622
|
-
if self._context.state < SupervisorStates.RUNNING:
|
3623
|
-
self._ordered_stop_groups_phase_2()
|
3624
|
-
|
3625
|
-
if self._context.test:
|
3626
|
-
break
|
3627
|
-
|
3628
|
-
def _tick(self, now: ta.Optional[float] = None) -> None:
|
3629
|
-
"""Send one or more 'tick' events when the timeslice related to the period for the event type rolls over"""
|
3630
|
-
|
3631
|
-
if now is None:
|
3632
|
-
# now won't be None in unit tests
|
3633
|
-
now = time.time()
|
3634
|
-
|
3635
|
-
for event in TICK_EVENTS:
|
3636
|
-
period = event.period # type: ignore
|
3637
|
-
|
3638
|
-
last_tick = self._ticks.get(period)
|
3639
|
-
if last_tick is None:
|
3640
|
-
# we just started up
|
3641
|
-
last_tick = self._ticks[period] = timeslice(period, now)
|
3840
|
+
except ExitNow:
|
3841
|
+
raise
|
3842
|
+
except Exception: # noqa
|
3843
|
+
combined_map[fd].handle_error()
|
3844
|
+
else:
|
3845
|
+
log.debug('unexpected write event from fd %r', fd)
|
3846
|
+
try:
|
3847
|
+
self._context.poller.unregister_writable(fd)
|
3848
|
+
except Exception: # noqa
|
3849
|
+
pass
|
3642
3850
|
|
3643
|
-
|
3644
|
-
|
3645
|
-
self._ticks[period] = this_tick
|
3646
|
-
notify_event(event(this_tick, self))
|
3851
|
+
for group in pgroups:
|
3852
|
+
group.transition()
|
3647
3853
|
|
3648
3854
|
def _reap(self, *, once: bool = False, depth: int = 0) -> None:
|
3649
3855
|
if depth >= 100:
|
@@ -3693,50 +3899,77 @@ class Supervisor:
|
|
3693
3899
|
else:
|
3694
3900
|
log.debug('received %s indicating nothing', signame(sig))
|
3695
3901
|
|
3902
|
+
def _tick(self, now: ta.Optional[float] = None) -> None:
|
3903
|
+
"""Send one or more 'tick' events when the timeslice related to the period for the event type rolls over"""
|
3904
|
+
|
3905
|
+
if now is None:
|
3906
|
+
# now won't be None in unit tests
|
3907
|
+
now = time.time()
|
3696
3908
|
|
3697
|
-
|
3698
|
-
|
3909
|
+
for event in TICK_EVENTS:
|
3910
|
+
period = event.period # type: ignore
|
3911
|
+
|
3912
|
+
last_tick = self._ticks.get(period)
|
3913
|
+
if last_tick is None:
|
3914
|
+
# we just started up
|
3915
|
+
last_tick = self._ticks[period] = timeslice(period, now)
|
3916
|
+
|
3917
|
+
this_tick = timeslice(period, now)
|
3918
|
+
if this_tick != last_tick:
|
3919
|
+
self._ticks[period] = this_tick
|
3920
|
+
EVENT_CALLBACKS.notify(event(this_tick, self))
|
3921
|
+
|
3922
|
+
|
3923
|
+
########################################
|
3924
|
+
# main.py
|
3699
3925
|
|
3700
3926
|
|
3701
|
-
def main(
|
3927
|
+
def main(
|
3928
|
+
argv: ta.Optional[ta.Sequence[str]] = None,
|
3929
|
+
*,
|
3930
|
+
no_logging: bool = False,
|
3931
|
+
) -> None:
|
3702
3932
|
import argparse
|
3703
3933
|
|
3704
3934
|
parser = argparse.ArgumentParser()
|
3705
3935
|
parser.add_argument('config_file', metavar='config-file')
|
3706
|
-
|
3936
|
+
parser.add_argument('--no-journald', action='store_true')
|
3937
|
+
args = parser.parse_args(argv)
|
3707
3938
|
|
3708
|
-
|
3939
|
+
#
|
3709
3940
|
|
3710
3941
|
if not (cf := args.config_file):
|
3711
3942
|
raise RuntimeError('No config file specified')
|
3712
3943
|
|
3944
|
+
if not no_logging:
|
3945
|
+
configure_standard_logging(
|
3946
|
+
'INFO',
|
3947
|
+
handler_factory=journald_log_handler_factory if not args.no_journald else None,
|
3948
|
+
)
|
3949
|
+
|
3950
|
+
#
|
3951
|
+
|
3713
3952
|
# if we hup, restart by making a new Supervisor()
|
3714
|
-
|
3715
|
-
while True:
|
3953
|
+
for epoch in itertools.count():
|
3716
3954
|
with open(cf) as f:
|
3717
3955
|
config_src = f.read()
|
3956
|
+
|
3718
3957
|
config_dct = json.loads(config_src)
|
3719
3958
|
config: ServerConfig = unmarshal_obj(config_dct, ServerConfig)
|
3720
3959
|
|
3721
3960
|
context = ServerContext(
|
3722
3961
|
config,
|
3962
|
+
epoch=epoch,
|
3723
3963
|
)
|
3724
3964
|
|
3725
|
-
|
3726
|
-
|
3727
|
-
|
3728
|
-
|
3729
|
-
|
3730
|
-
if test or (context.state < SupervisorStates.RESTARTING):
|
3731
|
-
break
|
3732
|
-
|
3965
|
+
supervisor = Supervisor(context)
|
3966
|
+
try:
|
3967
|
+
supervisor.main()
|
3968
|
+
except ExitNow:
|
3969
|
+
pass
|
3733
3970
|
|
3734
|
-
|
3735
|
-
|
3736
|
-
try:
|
3737
|
-
d.main()
|
3738
|
-
except ExitNow:
|
3739
|
-
pass
|
3971
|
+
if context.state < SupervisorStates.RESTARTING:
|
3972
|
+
break
|
3740
3973
|
|
3741
3974
|
|
3742
3975
|
if __name__ == '__main__':
|