ominfra 0.0.0.dev118__py3-none-any.whl → 0.0.0.dev119__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ominfra/deploy/_executor.py +6 -2
- ominfra/deploy/poly/_main.py +6 -2
- ominfra/journald/fields.py +187 -0
- ominfra/journald/tailer.py +375 -312
- ominfra/pyremote/_runcommands.py +6 -2
- ominfra/scripts/journald2aws.py +381 -314
- ominfra/scripts/supervisor.py +649 -425
- ominfra/supervisor/__main__.py +1 -1
- ominfra/supervisor/context.py +50 -25
- ominfra/supervisor/dispatchers.py +8 -9
- ominfra/supervisor/main.py +68 -0
- ominfra/supervisor/poller.py +1 -3
- ominfra/supervisor/process.py +2 -4
- ominfra/supervisor/supervisor.py +100 -141
- {ominfra-0.0.0.dev118.dist-info → ominfra-0.0.0.dev119.dist-info}/METADATA +3 -3
- {ominfra-0.0.0.dev118.dist-info → ominfra-0.0.0.dev119.dist-info}/RECORD +20 -18
- {ominfra-0.0.0.dev118.dist-info → ominfra-0.0.0.dev119.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev118.dist-info → ominfra-0.0.0.dev119.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev118.dist-info → ominfra-0.0.0.dev119.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev118.dist-info → ominfra-0.0.0.dev119.dist-info}/top_level.txt +0 -0
ominfra/scripts/supervisor.py
CHANGED
@@ -2,12 +2,13 @@
|
|
2
2
|
# noinspection DuplicatedCode
|
3
3
|
# @omlish-lite
|
4
4
|
# @omlish-script
|
5
|
-
# @omlish-amalg-output ../supervisor/
|
6
|
-
# ruff: noqa: N802 UP006 UP007 UP036
|
5
|
+
# @omlish-amalg-output ../supervisor/main.py
|
6
|
+
# ruff: noqa: N802 UP006 UP007 UP012 UP036
|
7
7
|
import abc
|
8
8
|
import base64
|
9
9
|
import collections.abc
|
10
10
|
import contextlib
|
11
|
+
import ctypes as ct
|
11
12
|
import dataclasses as dc
|
12
13
|
import datetime
|
13
14
|
import decimal
|
@@ -17,6 +18,7 @@ import fcntl
|
|
17
18
|
import fractions
|
18
19
|
import functools
|
19
20
|
import grp
|
21
|
+
import itertools
|
20
22
|
import json
|
21
23
|
import logging
|
22
24
|
import os
|
@@ -28,6 +30,7 @@ import shlex
|
|
28
30
|
import signal
|
29
31
|
import stat
|
30
32
|
import sys
|
33
|
+
import syslog
|
31
34
|
import tempfile
|
32
35
|
import threading
|
33
36
|
import time
|
@@ -466,240 +469,28 @@ class NoPermissionError(ProcessError):
|
|
466
469
|
|
467
470
|
|
468
471
|
########################################
|
469
|
-
#
|
470
|
-
|
471
|
-
|
472
|
-
log = logging.getLogger(__name__)
|
472
|
+
# ../../../omlish/lite/cached.py
|
473
473
|
|
474
474
|
|
475
|
-
class
|
476
|
-
|
477
|
-
def __init__(self) -> None:
|
475
|
+
class _cached_nullary: # noqa
|
476
|
+
def __init__(self, fn):
|
478
477
|
super().__init__()
|
478
|
+
self._fn = fn
|
479
|
+
self._value = self._missing = object()
|
480
|
+
functools.update_wrapper(self, fn)
|
479
481
|
|
480
|
-
|
481
|
-
|
482
|
-
|
483
|
-
|
484
|
-
@abc.abstractmethod
|
485
|
-
def register_writable(self, fd: int) -> None:
|
486
|
-
raise NotImplementedError
|
487
|
-
|
488
|
-
@abc.abstractmethod
|
489
|
-
def unregister_readable(self, fd: int) -> None:
|
490
|
-
raise NotImplementedError
|
482
|
+
def __call__(self, *args, **kwargs): # noqa
|
483
|
+
if self._value is self._missing:
|
484
|
+
self._value = self._fn()
|
485
|
+
return self._value
|
491
486
|
|
492
|
-
|
493
|
-
|
494
|
-
|
487
|
+
def __get__(self, instance, owner): # noqa
|
488
|
+
bound = instance.__dict__[self._fn.__name__] = self.__class__(self._fn.__get__(instance, owner))
|
489
|
+
return bound
|
495
490
|
|
496
|
-
@abc.abstractmethod
|
497
|
-
def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[int], ta.List[int]]:
|
498
|
-
raise NotImplementedError
|
499
491
|
|
500
|
-
|
501
|
-
|
502
|
-
|
503
|
-
def after_daemonize(self) -> None: # noqa
|
504
|
-
pass
|
505
|
-
|
506
|
-
def close(self) -> None: # noqa
|
507
|
-
pass
|
508
|
-
|
509
|
-
|
510
|
-
class SelectPoller(BasePoller):
|
511
|
-
|
512
|
-
def __init__(self) -> None:
|
513
|
-
super().__init__()
|
514
|
-
|
515
|
-
self._readables: ta.Set[int] = set()
|
516
|
-
self._writables: ta.Set[int] = set()
|
517
|
-
|
518
|
-
def register_readable(self, fd: int) -> None:
|
519
|
-
self._readables.add(fd)
|
520
|
-
|
521
|
-
def register_writable(self, fd: int) -> None:
|
522
|
-
self._writables.add(fd)
|
523
|
-
|
524
|
-
def unregister_readable(self, fd: int) -> None:
|
525
|
-
self._readables.discard(fd)
|
526
|
-
|
527
|
-
def unregister_writable(self, fd: int) -> None:
|
528
|
-
self._writables.discard(fd)
|
529
|
-
|
530
|
-
def unregister_all(self) -> None:
|
531
|
-
self._readables.clear()
|
532
|
-
self._writables.clear()
|
533
|
-
|
534
|
-
def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[int], ta.List[int]]:
|
535
|
-
try:
|
536
|
-
r, w, x = select.select(
|
537
|
-
self._readables,
|
538
|
-
self._writables,
|
539
|
-
[], timeout,
|
540
|
-
)
|
541
|
-
except OSError as err:
|
542
|
-
if err.args[0] == errno.EINTR:
|
543
|
-
log.debug('EINTR encountered in poll')
|
544
|
-
return [], []
|
545
|
-
if err.args[0] == errno.EBADF:
|
546
|
-
log.debug('EBADF encountered in poll')
|
547
|
-
self.unregister_all()
|
548
|
-
return [], []
|
549
|
-
raise
|
550
|
-
return r, w
|
551
|
-
|
552
|
-
|
553
|
-
class PollPoller(BasePoller):
|
554
|
-
_READ = select.POLLIN | select.POLLPRI | select.POLLHUP
|
555
|
-
_WRITE = select.POLLOUT
|
556
|
-
|
557
|
-
def __init__(self) -> None:
|
558
|
-
super().__init__()
|
559
|
-
|
560
|
-
self._poller = select.poll()
|
561
|
-
self._readables: set[int] = set()
|
562
|
-
self._writables: set[int] = set()
|
563
|
-
|
564
|
-
def register_readable(self, fd: int) -> None:
|
565
|
-
self._poller.register(fd, self._READ)
|
566
|
-
self._readables.add(fd)
|
567
|
-
|
568
|
-
def register_writable(self, fd: int) -> None:
|
569
|
-
self._poller.register(fd, self._WRITE)
|
570
|
-
self._writables.add(fd)
|
571
|
-
|
572
|
-
def unregister_readable(self, fd: int) -> None:
|
573
|
-
self._readables.discard(fd)
|
574
|
-
self._poller.unregister(fd)
|
575
|
-
if fd in self._writables:
|
576
|
-
self._poller.register(fd, self._WRITE)
|
577
|
-
|
578
|
-
def unregister_writable(self, fd: int) -> None:
|
579
|
-
self._writables.discard(fd)
|
580
|
-
self._poller.unregister(fd)
|
581
|
-
if fd in self._readables:
|
582
|
-
self._poller.register(fd, self._READ)
|
583
|
-
|
584
|
-
def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[int], ta.List[int]]:
|
585
|
-
fds = self._poll_fds(timeout) # type: ignore
|
586
|
-
readables, writables = [], []
|
587
|
-
for fd, eventmask in fds:
|
588
|
-
if self._ignore_invalid(fd, eventmask):
|
589
|
-
continue
|
590
|
-
if eventmask & self._READ:
|
591
|
-
readables.append(fd)
|
592
|
-
if eventmask & self._WRITE:
|
593
|
-
writables.append(fd)
|
594
|
-
return readables, writables
|
595
|
-
|
596
|
-
def _poll_fds(self, timeout: float) -> ta.List[ta.Tuple[int, int]]:
|
597
|
-
try:
|
598
|
-
return self._poller.poll(timeout * 1000)
|
599
|
-
except OSError as err:
|
600
|
-
if err.args[0] == errno.EINTR:
|
601
|
-
log.debug('EINTR encountered in poll')
|
602
|
-
return []
|
603
|
-
raise
|
604
|
-
|
605
|
-
def _ignore_invalid(self, fd: int, eventmask: int) -> bool:
|
606
|
-
if eventmask & select.POLLNVAL:
|
607
|
-
# POLLNVAL means `fd` value is invalid, not open. When a process quits it's `fd`s are closed so there is no
|
608
|
-
# more reason to keep this `fd` registered If the process restarts it's `fd`s are registered again.
|
609
|
-
self._poller.unregister(fd)
|
610
|
-
self._readables.discard(fd)
|
611
|
-
self._writables.discard(fd)
|
612
|
-
return True
|
613
|
-
return False
|
614
|
-
|
615
|
-
|
616
|
-
if sys.platform == 'darwin' or sys.platform.startswith('freebsd'):
|
617
|
-
class KqueuePoller(BasePoller):
|
618
|
-
max_events = 1000
|
619
|
-
|
620
|
-
def __init__(self) -> None:
|
621
|
-
super().__init__()
|
622
|
-
|
623
|
-
self._kqueue: ta.Optional[ta.Any] = select.kqueue()
|
624
|
-
self._readables: set[int] = set()
|
625
|
-
self._writables: set[int] = set()
|
626
|
-
|
627
|
-
def register_readable(self, fd: int) -> None:
|
628
|
-
self._readables.add(fd)
|
629
|
-
kevent = select.kevent(fd, filter=select.KQ_FILTER_READ, flags=select.KQ_EV_ADD)
|
630
|
-
self._kqueue_control(fd, kevent)
|
631
|
-
|
632
|
-
def register_writable(self, fd: int) -> None:
|
633
|
-
self._writables.add(fd)
|
634
|
-
kevent = select.kevent(fd, filter=select.KQ_FILTER_WRITE, flags=select.KQ_EV_ADD)
|
635
|
-
self._kqueue_control(fd, kevent)
|
636
|
-
|
637
|
-
def unregister_readable(self, fd: int) -> None:
|
638
|
-
kevent = select.kevent(fd, filter=select.KQ_FILTER_READ, flags=select.KQ_EV_DELETE)
|
639
|
-
self._readables.discard(fd)
|
640
|
-
self._kqueue_control(fd, kevent)
|
641
|
-
|
642
|
-
def unregister_writable(self, fd: int) -> None:
|
643
|
-
kevent = select.kevent(fd, filter=select.KQ_FILTER_WRITE, flags=select.KQ_EV_DELETE)
|
644
|
-
self._writables.discard(fd)
|
645
|
-
self._kqueue_control(fd, kevent)
|
646
|
-
|
647
|
-
def _kqueue_control(self, fd: int, kevent: 'select.kevent') -> None:
|
648
|
-
try:
|
649
|
-
self._kqueue.control([kevent], 0) # type: ignore
|
650
|
-
except OSError as error:
|
651
|
-
if error.errno == errno.EBADF:
|
652
|
-
log.debug('EBADF encountered in kqueue. Invalid file descriptor %s', fd)
|
653
|
-
else:
|
654
|
-
raise
|
655
|
-
|
656
|
-
def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[int], ta.List[int]]:
|
657
|
-
readables, writables = [], [] # type: ignore
|
658
|
-
|
659
|
-
try:
|
660
|
-
kevents = self._kqueue.control(None, self.max_events, timeout) # type: ignore
|
661
|
-
except OSError as error:
|
662
|
-
if error.errno == errno.EINTR:
|
663
|
-
log.debug('EINTR encountered in poll')
|
664
|
-
return readables, writables
|
665
|
-
raise
|
666
|
-
|
667
|
-
for kevent in kevents:
|
668
|
-
if kevent.filter == select.KQ_FILTER_READ:
|
669
|
-
readables.append(kevent.ident)
|
670
|
-
if kevent.filter == select.KQ_FILTER_WRITE:
|
671
|
-
writables.append(kevent.ident)
|
672
|
-
|
673
|
-
return readables, writables
|
674
|
-
|
675
|
-
def before_daemonize(self) -> None:
|
676
|
-
self.close()
|
677
|
-
|
678
|
-
def after_daemonize(self) -> None:
|
679
|
-
self._kqueue = select.kqueue()
|
680
|
-
for fd in self._readables:
|
681
|
-
self.register_readable(fd)
|
682
|
-
for fd in self._writables:
|
683
|
-
self.register_writable(fd)
|
684
|
-
|
685
|
-
def close(self) -> None:
|
686
|
-
self._kqueue.close() # type: ignore
|
687
|
-
self._kqueue = None
|
688
|
-
|
689
|
-
else:
|
690
|
-
KqueuePoller = None
|
691
|
-
|
692
|
-
|
693
|
-
Poller: ta.Type[BasePoller]
|
694
|
-
if (
|
695
|
-
sys.platform == 'darwin' or sys.platform.startswith('freebsd') and
|
696
|
-
hasattr(select, 'kqueue') and KqueuePoller is not None
|
697
|
-
):
|
698
|
-
Poller = KqueuePoller
|
699
|
-
elif hasattr(select, 'poll'):
|
700
|
-
Poller = PollPoller
|
701
|
-
else:
|
702
|
-
Poller = SelectPoller
|
492
|
+
def cached_nullary(fn: ta.Callable[..., T]) -> ta.Callable[..., T]:
|
493
|
+
return _cached_nullary(fn)
|
703
494
|
|
704
495
|
|
705
496
|
########################################
|
@@ -1012,6 +803,164 @@ def get_supervisor_state_description(code: SupervisorState) -> str:
|
|
1012
803
|
return check_not_none(_supervisor_states_by_code.get(code))
|
1013
804
|
|
1014
805
|
|
806
|
+
########################################
|
807
|
+
# ../../../omlish/lite/journald.py
|
808
|
+
|
809
|
+
|
810
|
+
##
|
811
|
+
|
812
|
+
|
813
|
+
class sd_iovec(ct.Structure): # noqa
|
814
|
+
pass
|
815
|
+
|
816
|
+
|
817
|
+
sd_iovec._fields_ = [
|
818
|
+
('iov_base', ct.c_void_p), # Pointer to data.
|
819
|
+
('iov_len', ct.c_size_t), # Length of data.
|
820
|
+
]
|
821
|
+
|
822
|
+
|
823
|
+
##
|
824
|
+
|
825
|
+
|
826
|
+
@cached_nullary
|
827
|
+
def sd_libsystemd() -> ta.Any:
|
828
|
+
lib = ct.CDLL('libsystemd.so.0')
|
829
|
+
|
830
|
+
lib.sd_journal_sendv = lib['sd_journal_sendv'] # type: ignore
|
831
|
+
lib.sd_journal_sendv.restype = ct.c_int
|
832
|
+
lib.sd_journal_sendv.argtypes = [ct.POINTER(sd_iovec), ct.c_int]
|
833
|
+
|
834
|
+
return lib
|
835
|
+
|
836
|
+
|
837
|
+
@cached_nullary
|
838
|
+
def sd_try_libsystemd() -> ta.Optional[ta.Any]:
|
839
|
+
try:
|
840
|
+
return sd_libsystemd()
|
841
|
+
except OSError: # noqa
|
842
|
+
return None
|
843
|
+
|
844
|
+
|
845
|
+
##
|
846
|
+
|
847
|
+
|
848
|
+
def sd_journald_send(**fields: str) -> int:
|
849
|
+
lib = sd_libsystemd()
|
850
|
+
|
851
|
+
msgs = [
|
852
|
+
f'{k.upper()}={v}\0'.encode('utf-8')
|
853
|
+
for k, v in fields.items()
|
854
|
+
]
|
855
|
+
|
856
|
+
vec = (sd_iovec * len(msgs))()
|
857
|
+
cl = (ct.c_char_p * len(msgs))() # noqa
|
858
|
+
for i in range(len(msgs)):
|
859
|
+
vec[i].iov_base = ct.cast(ct.c_char_p(msgs[i]), ct.c_void_p)
|
860
|
+
vec[i].iov_len = len(msgs[i]) - 1
|
861
|
+
|
862
|
+
return lib.sd_journal_sendv(vec, len(msgs))
|
863
|
+
|
864
|
+
|
865
|
+
##
|
866
|
+
|
867
|
+
|
868
|
+
SD_LOG_LEVEL_MAP: ta.Mapping[int, int] = {
|
869
|
+
logging.FATAL: syslog.LOG_EMERG, # system is unusable
|
870
|
+
# LOG_ALERT ? # action must be taken immediately
|
871
|
+
logging.CRITICAL: syslog.LOG_CRIT,
|
872
|
+
logging.ERROR: syslog.LOG_ERR,
|
873
|
+
logging.WARNING: syslog.LOG_WARNING,
|
874
|
+
# LOG_NOTICE ? # normal but significant condition
|
875
|
+
logging.INFO: syslog.LOG_INFO,
|
876
|
+
logging.DEBUG: syslog.LOG_DEBUG,
|
877
|
+
}
|
878
|
+
|
879
|
+
|
880
|
+
class JournaldLogHandler(logging.Handler):
|
881
|
+
"""
|
882
|
+
TODO:
|
883
|
+
- fallback handler for when this barfs
|
884
|
+
"""
|
885
|
+
|
886
|
+
def __init__(
|
887
|
+
self,
|
888
|
+
*,
|
889
|
+
use_formatter_output: bool = False,
|
890
|
+
) -> None:
|
891
|
+
super().__init__()
|
892
|
+
|
893
|
+
sd_libsystemd()
|
894
|
+
|
895
|
+
self._use_formatter_output = use_formatter_output
|
896
|
+
|
897
|
+
#
|
898
|
+
|
899
|
+
EXTRA_RECORD_ATTRS_BY_JOURNALD_FIELD: ta.ClassVar[ta.Mapping[str, str]] = {
|
900
|
+
'name': 'name',
|
901
|
+
'module': 'module',
|
902
|
+
'exception': 'exc_text',
|
903
|
+
'thread_name': 'threadName',
|
904
|
+
'task_name': 'taskName',
|
905
|
+
}
|
906
|
+
|
907
|
+
def make_fields(self, record: logging.LogRecord) -> ta.Mapping[str, str]:
|
908
|
+
formatter_message = self.format(record)
|
909
|
+
if self._use_formatter_output:
|
910
|
+
message = formatter_message
|
911
|
+
else:
|
912
|
+
message = record.message
|
913
|
+
|
914
|
+
fields: dict[str, str] = {
|
915
|
+
'message': message,
|
916
|
+
'priority': str(SD_LOG_LEVEL_MAP[record.levelno]),
|
917
|
+
'tid': str(threading.get_ident()),
|
918
|
+
}
|
919
|
+
|
920
|
+
if (pathname := record.pathname) is not None:
|
921
|
+
fields['code_file'] = pathname
|
922
|
+
if (lineno := record.lineno) is not None:
|
923
|
+
fields['code_lineno'] = str(lineno)
|
924
|
+
if (func_name := record.funcName) is not None:
|
925
|
+
fields['code_func'] = func_name
|
926
|
+
|
927
|
+
for f, a in self.EXTRA_RECORD_ATTRS_BY_JOURNALD_FIELD.items():
|
928
|
+
if (v := getattr(record, a, None)) is not None:
|
929
|
+
fields[f] = str(v)
|
930
|
+
|
931
|
+
return fields
|
932
|
+
|
933
|
+
#
|
934
|
+
|
935
|
+
def emit(self, record: logging.LogRecord) -> None:
|
936
|
+
try:
|
937
|
+
fields = self.make_fields(record)
|
938
|
+
|
939
|
+
if rc := sd_journald_send(**fields):
|
940
|
+
raise RuntimeError(f'{self.__class__.__name__}.emit failed: {rc=}') # noqa
|
941
|
+
|
942
|
+
except RecursionError: # See issue 36272
|
943
|
+
raise
|
944
|
+
|
945
|
+
except Exception: # noqa
|
946
|
+
self.handleError(record)
|
947
|
+
|
948
|
+
|
949
|
+
def journald_log_handler_factory(
|
950
|
+
*,
|
951
|
+
no_tty_check: bool = False,
|
952
|
+
no_fallback: bool = False,
|
953
|
+
) -> logging.Handler:
|
954
|
+
if (
|
955
|
+
sys.platform == 'linux' and
|
956
|
+
(no_tty_check or not sys.stderr.isatty()) and
|
957
|
+
(no_fallback or sd_try_libsystemd() is not None)
|
958
|
+
):
|
959
|
+
return JournaldLogHandler()
|
960
|
+
|
961
|
+
return logging.StreamHandler()
|
962
|
+
|
963
|
+
|
1015
964
|
########################################
|
1016
965
|
# ../../../omlish/lite/logs.py
|
1017
966
|
"""
|
@@ -1099,7 +1048,7 @@ class StandardLogFormatter(logging.Formatter):
|
|
1099
1048
|
if datefmt:
|
1100
1049
|
return ct.strftime(datefmt) # noqa
|
1101
1050
|
else:
|
1102
|
-
t = ct.strftime(
|
1051
|
+
t = ct.strftime('%Y-%m-%d %H:%M:%S')
|
1103
1052
|
return '%s.%03d' % (t, record.msecs)
|
1104
1053
|
|
1105
1054
|
|
@@ -1236,6 +1185,7 @@ def configure_standard_logging(
|
|
1236
1185
|
json: bool = False,
|
1237
1186
|
target: ta.Optional[logging.Logger] = None,
|
1238
1187
|
force: bool = False,
|
1188
|
+
handler_factory: ta.Optional[ta.Callable[[], logging.Handler]] = None,
|
1239
1189
|
) -> ta.Optional[StandardLogHandler]:
|
1240
1190
|
with _locking_logging_module_lock():
|
1241
1191
|
if target is None:
|
@@ -1249,7 +1199,10 @@ def configure_standard_logging(
|
|
1249
1199
|
|
1250
1200
|
#
|
1251
1201
|
|
1252
|
-
|
1202
|
+
if handler_factory is not None:
|
1203
|
+
handler = handler_factory()
|
1204
|
+
else:
|
1205
|
+
handler = logging.StreamHandler()
|
1253
1206
|
|
1254
1207
|
#
|
1255
1208
|
|
@@ -1836,91 +1789,325 @@ class ProcessGroupEvent(Event):
|
|
1836
1789
|
return f'groupname:{self.group}\n'
|
1837
1790
|
|
1838
1791
|
|
1839
|
-
class ProcessGroupAddedEvent(ProcessGroupEvent):
|
1840
|
-
pass
|
1792
|
+
class ProcessGroupAddedEvent(ProcessGroupEvent):
|
1793
|
+
pass
|
1794
|
+
|
1795
|
+
|
1796
|
+
class ProcessGroupRemovedEvent(ProcessGroupEvent):
|
1797
|
+
pass
|
1798
|
+
|
1799
|
+
|
1800
|
+
class TickEvent(Event):
|
1801
|
+
""" Abstract """
|
1802
|
+
|
1803
|
+
def __init__(self, when, supervisord):
|
1804
|
+
super().__init__()
|
1805
|
+
self.when = when
|
1806
|
+
self.supervisord = supervisord
|
1807
|
+
|
1808
|
+
def payload(self):
|
1809
|
+
return f'when:{self.when}'
|
1810
|
+
|
1811
|
+
|
1812
|
+
class Tick5Event(TickEvent):
|
1813
|
+
period = 5
|
1814
|
+
|
1815
|
+
|
1816
|
+
class Tick60Event(TickEvent):
|
1817
|
+
period = 60
|
1818
|
+
|
1819
|
+
|
1820
|
+
class Tick3600Event(TickEvent):
|
1821
|
+
period = 3600
|
1822
|
+
|
1823
|
+
|
1824
|
+
TICK_EVENTS = [ # imported elsewhere
|
1825
|
+
Tick5Event,
|
1826
|
+
Tick60Event,
|
1827
|
+
Tick3600Event,
|
1828
|
+
]
|
1829
|
+
|
1830
|
+
|
1831
|
+
class EventTypes:
|
1832
|
+
EVENT = Event # abstract
|
1833
|
+
|
1834
|
+
PROCESS_STATE = ProcessStateEvent # abstract
|
1835
|
+
PROCESS_STATE_STOPPED = ProcessStateStoppedEvent
|
1836
|
+
PROCESS_STATE_EXITED = ProcessStateExitedEvent
|
1837
|
+
PROCESS_STATE_STARTING = ProcessStateStartingEvent
|
1838
|
+
PROCESS_STATE_STOPPING = ProcessStateStoppingEvent
|
1839
|
+
PROCESS_STATE_BACKOFF = ProcessStateBackoffEvent
|
1840
|
+
PROCESS_STATE_FATAL = ProcessStateFatalEvent
|
1841
|
+
PROCESS_STATE_RUNNING = ProcessStateRunningEvent
|
1842
|
+
PROCESS_STATE_UNKNOWN = ProcessStateUnknownEvent
|
1843
|
+
|
1844
|
+
PROCESS_COMMUNICATION = ProcessCommunicationEvent # abstract
|
1845
|
+
PROCESS_COMMUNICATION_STDOUT = ProcessCommunicationStdoutEvent
|
1846
|
+
PROCESS_COMMUNICATION_STDERR = ProcessCommunicationStderrEvent
|
1847
|
+
|
1848
|
+
PROCESS_LOG = ProcessLogEvent
|
1849
|
+
PROCESS_LOG_STDOUT = ProcessLogStdoutEvent
|
1850
|
+
PROCESS_LOG_STDERR = ProcessLogStderrEvent
|
1851
|
+
|
1852
|
+
REMOTE_COMMUNICATION = RemoteCommunicationEvent
|
1853
|
+
|
1854
|
+
SUPERVISOR_STATE_CHANGE = SupervisorStateChangeEvent # abstract
|
1855
|
+
SUPERVISOR_STATE_CHANGE_RUNNING = SupervisorRunningEvent
|
1856
|
+
SUPERVISOR_STATE_CHANGE_STOPPING = SupervisorStoppingEvent
|
1857
|
+
|
1858
|
+
TICK = TickEvent # abstract
|
1859
|
+
TICK_5 = Tick5Event
|
1860
|
+
TICK_60 = Tick60Event
|
1861
|
+
TICK_3600 = Tick3600Event
|
1862
|
+
|
1863
|
+
PROCESS_GROUP = ProcessGroupEvent # abstract
|
1864
|
+
PROCESS_GROUP_ADDED = ProcessGroupAddedEvent
|
1865
|
+
PROCESS_GROUP_REMOVED = ProcessGroupRemovedEvent
|
1866
|
+
|
1867
|
+
|
1868
|
+
def get_event_name_by_type(requested):
|
1869
|
+
for name, typ in EventTypes.__dict__.items():
|
1870
|
+
if typ is requested:
|
1871
|
+
return name
|
1872
|
+
return None
|
1873
|
+
|
1874
|
+
|
1875
|
+
def register(name, event):
|
1876
|
+
setattr(EventTypes, name, event)
|
1877
|
+
|
1878
|
+
|
1879
|
+
########################################
|
1880
|
+
# ../poller.py
|
1881
|
+
|
1882
|
+
|
1883
|
+
class BasePoller(abc.ABC):
|
1884
|
+
|
1885
|
+
def __init__(self) -> None:
|
1886
|
+
super().__init__()
|
1887
|
+
|
1888
|
+
@abc.abstractmethod
|
1889
|
+
def register_readable(self, fd: int) -> None:
|
1890
|
+
raise NotImplementedError
|
1891
|
+
|
1892
|
+
@abc.abstractmethod
|
1893
|
+
def register_writable(self, fd: int) -> None:
|
1894
|
+
raise NotImplementedError
|
1895
|
+
|
1896
|
+
@abc.abstractmethod
|
1897
|
+
def unregister_readable(self, fd: int) -> None:
|
1898
|
+
raise NotImplementedError
|
1899
|
+
|
1900
|
+
@abc.abstractmethod
|
1901
|
+
def unregister_writable(self, fd: int) -> None:
|
1902
|
+
raise NotImplementedError
|
1903
|
+
|
1904
|
+
@abc.abstractmethod
|
1905
|
+
def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[int], ta.List[int]]:
|
1906
|
+
raise NotImplementedError
|
1907
|
+
|
1908
|
+
def before_daemonize(self) -> None: # noqa
|
1909
|
+
pass
|
1910
|
+
|
1911
|
+
def after_daemonize(self) -> None: # noqa
|
1912
|
+
pass
|
1913
|
+
|
1914
|
+
def close(self) -> None: # noqa
|
1915
|
+
pass
|
1916
|
+
|
1917
|
+
|
1918
|
+
class SelectPoller(BasePoller):
|
1919
|
+
|
1920
|
+
def __init__(self) -> None:
|
1921
|
+
super().__init__()
|
1922
|
+
|
1923
|
+
self._readables: ta.Set[int] = set()
|
1924
|
+
self._writables: ta.Set[int] = set()
|
1925
|
+
|
1926
|
+
def register_readable(self, fd: int) -> None:
|
1927
|
+
self._readables.add(fd)
|
1928
|
+
|
1929
|
+
def register_writable(self, fd: int) -> None:
|
1930
|
+
self._writables.add(fd)
|
1931
|
+
|
1932
|
+
def unregister_readable(self, fd: int) -> None:
|
1933
|
+
self._readables.discard(fd)
|
1934
|
+
|
1935
|
+
def unregister_writable(self, fd: int) -> None:
|
1936
|
+
self._writables.discard(fd)
|
1937
|
+
|
1938
|
+
def unregister_all(self) -> None:
|
1939
|
+
self._readables.clear()
|
1940
|
+
self._writables.clear()
|
1941
|
+
|
1942
|
+
def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[int], ta.List[int]]:
|
1943
|
+
try:
|
1944
|
+
r, w, x = select.select(
|
1945
|
+
self._readables,
|
1946
|
+
self._writables,
|
1947
|
+
[], timeout,
|
1948
|
+
)
|
1949
|
+
except OSError as err:
|
1950
|
+
if err.args[0] == errno.EINTR:
|
1951
|
+
log.debug('EINTR encountered in poll')
|
1952
|
+
return [], []
|
1953
|
+
if err.args[0] == errno.EBADF:
|
1954
|
+
log.debug('EBADF encountered in poll')
|
1955
|
+
self.unregister_all()
|
1956
|
+
return [], []
|
1957
|
+
raise
|
1958
|
+
return r, w
|
1959
|
+
|
1960
|
+
|
1961
|
+
class PollPoller(BasePoller):
|
1962
|
+
_READ = select.POLLIN | select.POLLPRI | select.POLLHUP
|
1963
|
+
_WRITE = select.POLLOUT
|
1964
|
+
|
1965
|
+
def __init__(self) -> None:
|
1966
|
+
super().__init__()
|
1841
1967
|
|
1968
|
+
self._poller = select.poll()
|
1969
|
+
self._readables: set[int] = set()
|
1970
|
+
self._writables: set[int] = set()
|
1842
1971
|
|
1843
|
-
|
1844
|
-
|
1972
|
+
def register_readable(self, fd: int) -> None:
|
1973
|
+
self._poller.register(fd, self._READ)
|
1974
|
+
self._readables.add(fd)
|
1845
1975
|
|
1976
|
+
def register_writable(self, fd: int) -> None:
|
1977
|
+
self._poller.register(fd, self._WRITE)
|
1978
|
+
self._writables.add(fd)
|
1846
1979
|
|
1847
|
-
|
1848
|
-
|
1980
|
+
def unregister_readable(self, fd: int) -> None:
|
1981
|
+
self._readables.discard(fd)
|
1982
|
+
self._poller.unregister(fd)
|
1983
|
+
if fd in self._writables:
|
1984
|
+
self._poller.register(fd, self._WRITE)
|
1849
1985
|
|
1850
|
-
def
|
1851
|
-
|
1852
|
-
self.
|
1853
|
-
self.
|
1986
|
+
def unregister_writable(self, fd: int) -> None:
|
1987
|
+
self._writables.discard(fd)
|
1988
|
+
self._poller.unregister(fd)
|
1989
|
+
if fd in self._readables:
|
1990
|
+
self._poller.register(fd, self._READ)
|
1854
1991
|
|
1855
|
-
def
|
1856
|
-
|
1992
|
+
def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[int], ta.List[int]]:
|
1993
|
+
fds = self._poll_fds(timeout) # type: ignore
|
1994
|
+
readables, writables = [], []
|
1995
|
+
for fd, eventmask in fds:
|
1996
|
+
if self._ignore_invalid(fd, eventmask):
|
1997
|
+
continue
|
1998
|
+
if eventmask & self._READ:
|
1999
|
+
readables.append(fd)
|
2000
|
+
if eventmask & self._WRITE:
|
2001
|
+
writables.append(fd)
|
2002
|
+
return readables, writables
|
1857
2003
|
|
2004
|
+
def _poll_fds(self, timeout: float) -> ta.List[ta.Tuple[int, int]]:
|
2005
|
+
try:
|
2006
|
+
return self._poller.poll(timeout * 1000)
|
2007
|
+
except OSError as err:
|
2008
|
+
if err.args[0] == errno.EINTR:
|
2009
|
+
log.debug('EINTR encountered in poll')
|
2010
|
+
return []
|
2011
|
+
raise
|
1858
2012
|
|
1859
|
-
|
1860
|
-
|
2013
|
+
def _ignore_invalid(self, fd: int, eventmask: int) -> bool:
|
2014
|
+
if eventmask & select.POLLNVAL:
|
2015
|
+
# POLLNVAL means `fd` value is invalid, not open. When a process quits it's `fd`s are closed so there is no
|
2016
|
+
# more reason to keep this `fd` registered If the process restarts it's `fd`s are registered again.
|
2017
|
+
self._poller.unregister(fd)
|
2018
|
+
self._readables.discard(fd)
|
2019
|
+
self._writables.discard(fd)
|
2020
|
+
return True
|
2021
|
+
return False
|
1861
2022
|
|
1862
2023
|
|
1863
|
-
|
1864
|
-
|
2024
|
+
if sys.platform == 'darwin' or sys.platform.startswith('freebsd'):
|
2025
|
+
class KqueuePoller(BasePoller):
|
2026
|
+
max_events = 1000
|
1865
2027
|
|
2028
|
+
def __init__(self) -> None:
|
2029
|
+
super().__init__()
|
1866
2030
|
|
1867
|
-
|
1868
|
-
|
2031
|
+
self._kqueue: ta.Optional[ta.Any] = select.kqueue()
|
2032
|
+
self._readables: set[int] = set()
|
2033
|
+
self._writables: set[int] = set()
|
1869
2034
|
|
2035
|
+
def register_readable(self, fd: int) -> None:
|
2036
|
+
self._readables.add(fd)
|
2037
|
+
kevent = select.kevent(fd, filter=select.KQ_FILTER_READ, flags=select.KQ_EV_ADD)
|
2038
|
+
self._kqueue_control(fd, kevent)
|
1870
2039
|
|
1871
|
-
|
1872
|
-
|
1873
|
-
|
1874
|
-
|
1875
|
-
]
|
2040
|
+
def register_writable(self, fd: int) -> None:
|
2041
|
+
self._writables.add(fd)
|
2042
|
+
kevent = select.kevent(fd, filter=select.KQ_FILTER_WRITE, flags=select.KQ_EV_ADD)
|
2043
|
+
self._kqueue_control(fd, kevent)
|
1876
2044
|
|
2045
|
+
def unregister_readable(self, fd: int) -> None:
|
2046
|
+
kevent = select.kevent(fd, filter=select.KQ_FILTER_READ, flags=select.KQ_EV_DELETE)
|
2047
|
+
self._readables.discard(fd)
|
2048
|
+
self._kqueue_control(fd, kevent)
|
1877
2049
|
|
1878
|
-
|
1879
|
-
|
2050
|
+
def unregister_writable(self, fd: int) -> None:
|
2051
|
+
kevent = select.kevent(fd, filter=select.KQ_FILTER_WRITE, flags=select.KQ_EV_DELETE)
|
2052
|
+
self._writables.discard(fd)
|
2053
|
+
self._kqueue_control(fd, kevent)
|
1880
2054
|
|
1881
|
-
|
1882
|
-
|
1883
|
-
|
1884
|
-
|
1885
|
-
|
1886
|
-
|
1887
|
-
|
1888
|
-
|
1889
|
-
PROCESS_STATE_UNKNOWN = ProcessStateUnknownEvent
|
2055
|
+
def _kqueue_control(self, fd: int, kevent: 'select.kevent') -> None:
|
2056
|
+
try:
|
2057
|
+
self._kqueue.control([kevent], 0) # type: ignore
|
2058
|
+
except OSError as error:
|
2059
|
+
if error.errno == errno.EBADF:
|
2060
|
+
log.debug('EBADF encountered in kqueue. Invalid file descriptor %s', fd)
|
2061
|
+
else:
|
2062
|
+
raise
|
1890
2063
|
|
1891
|
-
|
1892
|
-
|
1893
|
-
PROCESS_COMMUNICATION_STDERR = ProcessCommunicationStderrEvent
|
2064
|
+
def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[int], ta.List[int]]:
|
2065
|
+
readables, writables = [], [] # type: ignore
|
1894
2066
|
|
1895
|
-
|
1896
|
-
|
1897
|
-
|
2067
|
+
try:
|
2068
|
+
kevents = self._kqueue.control(None, self.max_events, timeout) # type: ignore
|
2069
|
+
except OSError as error:
|
2070
|
+
if error.errno == errno.EINTR:
|
2071
|
+
log.debug('EINTR encountered in poll')
|
2072
|
+
return readables, writables
|
2073
|
+
raise
|
1898
2074
|
|
1899
|
-
|
2075
|
+
for kevent in kevents:
|
2076
|
+
if kevent.filter == select.KQ_FILTER_READ:
|
2077
|
+
readables.append(kevent.ident)
|
2078
|
+
if kevent.filter == select.KQ_FILTER_WRITE:
|
2079
|
+
writables.append(kevent.ident)
|
1900
2080
|
|
1901
|
-
|
1902
|
-
SUPERVISOR_STATE_CHANGE_RUNNING = SupervisorRunningEvent
|
1903
|
-
SUPERVISOR_STATE_CHANGE_STOPPING = SupervisorStoppingEvent
|
2081
|
+
return readables, writables
|
1904
2082
|
|
1905
|
-
|
1906
|
-
|
1907
|
-
TICK_60 = Tick60Event
|
1908
|
-
TICK_3600 = Tick3600Event
|
2083
|
+
def before_daemonize(self) -> None:
|
2084
|
+
self.close()
|
1909
2085
|
|
1910
|
-
|
1911
|
-
|
1912
|
-
|
2086
|
+
def after_daemonize(self) -> None:
|
2087
|
+
self._kqueue = select.kqueue()
|
2088
|
+
for fd in self._readables:
|
2089
|
+
self.register_readable(fd)
|
2090
|
+
for fd in self._writables:
|
2091
|
+
self.register_writable(fd)
|
1913
2092
|
|
2093
|
+
def close(self) -> None:
|
2094
|
+
self._kqueue.close() # type: ignore
|
2095
|
+
self._kqueue = None
|
1914
2096
|
|
1915
|
-
|
1916
|
-
|
1917
|
-
if typ is requested:
|
1918
|
-
return name
|
1919
|
-
return None
|
2097
|
+
else:
|
2098
|
+
KqueuePoller = None
|
1920
2099
|
|
1921
2100
|
|
1922
|
-
|
1923
|
-
|
2101
|
+
Poller: ta.Type[BasePoller]
|
2102
|
+
if (
|
2103
|
+
sys.platform == 'darwin' or sys.platform.startswith('freebsd') and
|
2104
|
+
hasattr(select, 'kqueue') and KqueuePoller is not None
|
2105
|
+
):
|
2106
|
+
Poller = KqueuePoller
|
2107
|
+
elif hasattr(select, 'poll'):
|
2108
|
+
Poller = PollPoller
|
2109
|
+
else:
|
2110
|
+
Poller = SelectPoller
|
1924
2111
|
|
1925
2112
|
|
1926
2113
|
########################################
|
@@ -1973,41 +2160,47 @@ class AbstractSubprocess(abc.ABC):
|
|
1973
2160
|
# ../context.py
|
1974
2161
|
|
1975
2162
|
|
1976
|
-
log = logging.getLogger(__name__)
|
1977
|
-
|
1978
|
-
|
1979
2163
|
class ServerContext(AbstractServerContext):
|
1980
|
-
|
1981
|
-
|
1982
|
-
|
1983
|
-
|
1984
|
-
|
1985
|
-
|
2164
|
+
def __init__(
|
2165
|
+
self,
|
2166
|
+
config: ServerConfig,
|
2167
|
+
*,
|
2168
|
+
epoch: int = 0,
|
2169
|
+
) -> None:
|
1986
2170
|
super().__init__()
|
1987
2171
|
|
1988
2172
|
self._config = config
|
2173
|
+
self._epoch = epoch
|
1989
2174
|
|
1990
2175
|
self._pid_history: ta.Dict[int, AbstractSubprocess] = {}
|
1991
2176
|
self._state: SupervisorState = SupervisorStates.RUNNING
|
1992
2177
|
|
1993
|
-
self.
|
2178
|
+
self._signal_receiver = SignalReceiver()
|
1994
2179
|
|
1995
|
-
self.
|
2180
|
+
self._poller: BasePoller = Poller()
|
1996
2181
|
|
1997
|
-
if
|
1998
|
-
uid = name_to_uid(
|
1999
|
-
self.
|
2000
|
-
self.
|
2182
|
+
if config.user is not None:
|
2183
|
+
uid = name_to_uid(config.user)
|
2184
|
+
self._uid: ta.Optional[int] = uid
|
2185
|
+
self._gid: ta.Optional[int] = gid_for_uid(uid)
|
2001
2186
|
else:
|
2002
|
-
self.
|
2003
|
-
self.
|
2187
|
+
self._uid = None
|
2188
|
+
self._gid = None
|
2004
2189
|
|
2005
|
-
self.
|
2190
|
+
self._unlink_pidfile = False
|
2006
2191
|
|
2007
2192
|
@property
|
2008
2193
|
def config(self) -> ServerConfig:
|
2009
2194
|
return self._config
|
2010
2195
|
|
2196
|
+
@property
|
2197
|
+
def epoch(self) -> int:
|
2198
|
+
return self._epoch
|
2199
|
+
|
2200
|
+
@property
|
2201
|
+
def first(self) -> bool:
|
2202
|
+
return not self._epoch
|
2203
|
+
|
2011
2204
|
@property
|
2012
2205
|
def state(self) -> SupervisorState:
|
2013
2206
|
return self._state
|
@@ -2015,17 +2208,26 @@ class ServerContext(AbstractServerContext):
|
|
2015
2208
|
def set_state(self, state: SupervisorState) -> None:
|
2016
2209
|
self._state = state
|
2017
2210
|
|
2211
|
+
@property
|
2212
|
+
def poller(self) -> BasePoller:
|
2213
|
+
return self._poller
|
2214
|
+
|
2018
2215
|
@property
|
2019
2216
|
def pid_history(self) -> ta.Dict[int, AbstractSubprocess]:
|
2020
2217
|
return self._pid_history
|
2021
2218
|
|
2022
|
-
|
2023
|
-
|
2219
|
+
@property
|
2220
|
+
def uid(self) -> ta.Optional[int]:
|
2221
|
+
return self._uid
|
2222
|
+
|
2223
|
+
@property
|
2224
|
+
def gid(self) -> ta.Optional[int]:
|
2225
|
+
return self._gid
|
2024
2226
|
|
2025
2227
|
##
|
2026
2228
|
|
2027
2229
|
def set_signals(self) -> None:
|
2028
|
-
self.
|
2230
|
+
self._signal_receiver.install(
|
2029
2231
|
signal.SIGTERM,
|
2030
2232
|
signal.SIGINT,
|
2031
2233
|
signal.SIGQUIT,
|
@@ -2136,7 +2338,7 @@ class ServerContext(AbstractServerContext):
|
|
2136
2338
|
))
|
2137
2339
|
|
2138
2340
|
def cleanup(self) -> None:
|
2139
|
-
if self.
|
2341
|
+
if self._unlink_pidfile:
|
2140
2342
|
try_unlink(self.config.pidfile)
|
2141
2343
|
self.poller.close()
|
2142
2344
|
|
@@ -2189,6 +2391,7 @@ class ServerContext(AbstractServerContext):
|
|
2189
2391
|
# Parent
|
2190
2392
|
log.debug('supervisord forked; parent exiting')
|
2191
2393
|
real_exit(0)
|
2394
|
+
|
2192
2395
|
# Child
|
2193
2396
|
log.info('daemonizing the supervisord process')
|
2194
2397
|
if self.config.directory:
|
@@ -2198,11 +2401,15 @@ class ServerContext(AbstractServerContext):
|
|
2198
2401
|
log.critical("can't chdir into %r: %s", self.config.directory, err)
|
2199
2402
|
else:
|
2200
2403
|
log.info('set current directory: %r', self.config.directory)
|
2404
|
+
|
2201
2405
|
os.dup2(0, os.open('/dev/null', os.O_RDONLY))
|
2202
2406
|
os.dup2(1, os.open('/dev/null', os.O_WRONLY))
|
2203
2407
|
os.dup2(2, os.open('/dev/null', os.O_WRONLY))
|
2408
|
+
|
2204
2409
|
os.setsid()
|
2410
|
+
|
2205
2411
|
os.umask(self.config.umask)
|
2412
|
+
|
2206
2413
|
# XXX Stevens, in his Advanced Unix book, section 13.3 (page 417) recommends calling umask(0) and closing unused
|
2207
2414
|
# file descriptors. In his Network Programming book, he additionally recommends ignoring SIGHUP and forking
|
2208
2415
|
# again after the setsid() call, for obscure SVR4 reasons.
|
@@ -2217,7 +2424,7 @@ class ServerContext(AbstractServerContext):
|
|
2217
2424
|
return logfile
|
2218
2425
|
|
2219
2426
|
def get_signal(self) -> ta.Optional[int]:
|
2220
|
-
return self.
|
2427
|
+
return self._signal_receiver.get_signal()
|
2221
2428
|
|
2222
2429
|
def write_pidfile(self) -> None:
|
2223
2430
|
pid = os.getpid()
|
@@ -2227,7 +2434,7 @@ class ServerContext(AbstractServerContext):
|
|
2227
2434
|
except OSError:
|
2228
2435
|
log.critical('could not write pidfile %s', self.config.pidfile)
|
2229
2436
|
else:
|
2230
|
-
self.
|
2437
|
+
self._unlink_pidfile = True
|
2231
2438
|
log.info('supervisord started with pid %s', pid)
|
2232
2439
|
|
2233
2440
|
|
@@ -2280,11 +2487,14 @@ def drop_privileges(user: ta.Union[int, str, None]) -> ta.Optional[str]:
|
|
2280
2487
|
os.setgroups(groups)
|
2281
2488
|
except OSError:
|
2282
2489
|
return 'Could not set groups of effective user'
|
2490
|
+
|
2283
2491
|
try:
|
2284
2492
|
os.setgid(gid)
|
2285
2493
|
except OSError:
|
2286
2494
|
return 'Could not set group id of effective user'
|
2495
|
+
|
2287
2496
|
os.setuid(uid)
|
2497
|
+
|
2288
2498
|
return None
|
2289
2499
|
|
2290
2500
|
|
@@ -2354,9 +2564,6 @@ def check_execv_args(filename, argv, st) -> None:
|
|
2354
2564
|
# ../dispatchers.py
|
2355
2565
|
|
2356
2566
|
|
2357
|
-
log = logging.getLogger(__name__)
|
2358
|
-
|
2359
|
-
|
2360
2567
|
class Dispatcher(abc.ABC):
|
2361
2568
|
|
2362
2569
|
def __init__(self, process: AbstractSubprocess, channel: str, fd: int) -> None:
|
@@ -2513,16 +2720,16 @@ class OutputDispatcher(Dispatcher):
|
|
2513
2720
|
# )
|
2514
2721
|
|
2515
2722
|
def remove_logs(self):
|
2516
|
-
for
|
2517
|
-
if
|
2518
|
-
for handler in
|
2723
|
+
for l in (self._normal_log, self._capture_log):
|
2724
|
+
if l is not None:
|
2725
|
+
for handler in l.handlers:
|
2519
2726
|
handler.remove() # type: ignore
|
2520
2727
|
handler.reopen() # type: ignore
|
2521
2728
|
|
2522
2729
|
def reopen_logs(self):
|
2523
|
-
for
|
2524
|
-
if
|
2525
|
-
for handler in
|
2730
|
+
for l in (self._normal_log, self._capture_log):
|
2731
|
+
if l is not None:
|
2732
|
+
for handler in l.handlers:
|
2526
2733
|
handler.reopen() # type: ignore
|
2527
2734
|
|
2528
2735
|
def _log(self, data):
|
@@ -2660,9 +2867,6 @@ class InputDispatcher(Dispatcher):
|
|
2660
2867
|
# ../process.py
|
2661
2868
|
|
2662
2869
|
|
2663
|
-
log = logging.getLogger(__name__)
|
2664
|
-
|
2665
|
-
|
2666
2870
|
@functools.total_ordering
|
2667
2871
|
class Subprocess(AbstractSubprocess):
|
2668
2872
|
"""A class to manage a subprocess."""
|
@@ -3397,10 +3601,11 @@ class ProcessGroup:
|
|
3397
3601
|
|
3398
3602
|
|
3399
3603
|
########################################
|
3400
|
-
# supervisor.py
|
3604
|
+
# ../supervisor.py
|
3401
3605
|
|
3402
3606
|
|
3403
|
-
|
3607
|
+
def timeslice(period, when):
|
3608
|
+
return int(when - (when % period))
|
3404
3609
|
|
3405
3610
|
|
3406
3611
|
class Supervisor:
|
@@ -3423,6 +3628,11 @@ class Supervisor:
|
|
3423
3628
|
return self._context.state
|
3424
3629
|
|
3425
3630
|
def main(self) -> None:
|
3631
|
+
self.setup()
|
3632
|
+
self.run()
|
3633
|
+
|
3634
|
+
@cached_nullary
|
3635
|
+
def setup(self) -> None:
|
3426
3636
|
if not self._context.first:
|
3427
3637
|
# prevent crash on libdispatch-based systems, at least for the first request
|
3428
3638
|
self._context.cleanup_fds()
|
@@ -3437,9 +3647,11 @@ class Supervisor:
|
|
3437
3647
|
# clean up old automatic logs
|
3438
3648
|
self._context.clear_auto_child_logdir()
|
3439
3649
|
|
3440
|
-
|
3441
|
-
|
3442
|
-
|
3650
|
+
def run(
|
3651
|
+
self,
|
3652
|
+
*,
|
3653
|
+
callback: ta.Optional[ta.Callable[['Supervisor'], bool]] = None,
|
3654
|
+
) -> None:
|
3443
3655
|
self._process_groups = {} # clear
|
3444
3656
|
self._stop_groups = None # clear
|
3445
3657
|
|
@@ -3457,12 +3669,23 @@ class Supervisor:
|
|
3457
3669
|
# writing pid file needs to come *after* daemonizing or pid will be wrong
|
3458
3670
|
self._context.write_pidfile()
|
3459
3671
|
|
3460
|
-
|
3672
|
+
notify_event(SupervisorRunningEvent())
|
3673
|
+
|
3674
|
+
while True:
|
3675
|
+
if callback is not None and not callback(self):
|
3676
|
+
break
|
3677
|
+
|
3678
|
+
self._run_once()
|
3461
3679
|
|
3462
3680
|
finally:
|
3463
3681
|
self._context.cleanup()
|
3464
3682
|
|
3465
|
-
|
3683
|
+
class DiffToActive(ta.NamedTuple):
|
3684
|
+
added: ta.List[ProcessGroupConfig]
|
3685
|
+
changed: ta.List[ProcessGroupConfig]
|
3686
|
+
removed: ta.List[ProcessGroupConfig]
|
3687
|
+
|
3688
|
+
def diff_to_active(self) -> DiffToActive:
|
3466
3689
|
new = self._context.config.groups or []
|
3467
3690
|
cur = [group.config for group in self._process_groups.values()]
|
3468
3691
|
|
@@ -3474,7 +3697,7 @@ class Supervisor:
|
|
3474
3697
|
|
3475
3698
|
changed = [cand for cand in new if cand != curdict.get(cand.name, cand)]
|
3476
3699
|
|
3477
|
-
return added, changed, removed
|
3700
|
+
return Supervisor.DiffToActive(added, changed, removed)
|
3478
3701
|
|
3479
3702
|
def add_process_group(self, config: ProcessGroupConfig) -> bool:
|
3480
3703
|
name = config.name
|
@@ -3540,90 +3763,84 @@ class Supervisor:
|
|
3540
3763
|
# down, so push it back on to the end of the stop group queue
|
3541
3764
|
self._stop_groups.append(group)
|
3542
3765
|
|
3543
|
-
def
|
3544
|
-
|
3545
|
-
|
3546
|
-
|
3547
|
-
while True:
|
3548
|
-
combined_map = {}
|
3549
|
-
combined_map.update(self.get_process_map())
|
3766
|
+
def _run_once(self) -> None:
|
3767
|
+
combined_map = {}
|
3768
|
+
combined_map.update(self.get_process_map())
|
3550
3769
|
|
3551
|
-
|
3552
|
-
|
3770
|
+
pgroups = list(self._process_groups.values())
|
3771
|
+
pgroups.sort()
|
3553
3772
|
|
3554
|
-
|
3555
|
-
|
3556
|
-
|
3557
|
-
|
3558
|
-
|
3559
|
-
|
3773
|
+
if self._context.state < SupervisorStates.RUNNING:
|
3774
|
+
if not self._stopping:
|
3775
|
+
# first time, set the stopping flag, do a notification and set stop_groups
|
3776
|
+
self._stopping = True
|
3777
|
+
self._stop_groups = pgroups[:]
|
3778
|
+
notify_event(SupervisorStoppingEvent())
|
3560
3779
|
|
3561
|
-
|
3780
|
+
self._ordered_stop_groups_phase_1()
|
3562
3781
|
|
3563
|
-
|
3564
|
-
|
3565
|
-
|
3782
|
+
if not self.shutdown_report():
|
3783
|
+
# if there are no unstopped processes (we're done killing everything), it's OK to shutdown or reload
|
3784
|
+
raise ExitNow
|
3566
3785
|
|
3567
|
-
|
3568
|
-
|
3569
|
-
|
3570
|
-
|
3571
|
-
|
3786
|
+
for fd, dispatcher in combined_map.items():
|
3787
|
+
if dispatcher.readable():
|
3788
|
+
self._context.poller.register_readable(fd)
|
3789
|
+
if dispatcher.writable():
|
3790
|
+
self._context.poller.register_writable(fd)
|
3572
3791
|
|
3573
|
-
|
3792
|
+
timeout = 1 # this cannot be fewer than the smallest TickEvent (5)
|
3793
|
+
r, w = self._context.poller.poll(timeout)
|
3574
3794
|
|
3575
|
-
|
3576
|
-
|
3577
|
-
|
3578
|
-
|
3579
|
-
|
3580
|
-
|
3581
|
-
|
3582
|
-
self._context.poller.unregister_readable(fd)
|
3583
|
-
except ExitNow:
|
3584
|
-
raise
|
3585
|
-
except Exception: # noqa
|
3586
|
-
combined_map[fd].handle_error()
|
3587
|
-
else:
|
3588
|
-
# if the fd is not in combined_map, we should unregister it. otherwise, it will be polled every
|
3589
|
-
# time, which may cause 100% cpu usage
|
3590
|
-
log.debug('unexpected read event from fd %r', fd)
|
3591
|
-
try:
|
3795
|
+
for fd in r:
|
3796
|
+
if fd in combined_map:
|
3797
|
+
try:
|
3798
|
+
dispatcher = combined_map[fd]
|
3799
|
+
log.debug('read event caused by %r', dispatcher)
|
3800
|
+
dispatcher.handle_read_event()
|
3801
|
+
if not dispatcher.readable():
|
3592
3802
|
self._context.poller.unregister_readable(fd)
|
3593
|
-
|
3594
|
-
|
3803
|
+
except ExitNow:
|
3804
|
+
raise
|
3805
|
+
except Exception: # noqa
|
3806
|
+
combined_map[fd].handle_error()
|
3807
|
+
else:
|
3808
|
+
# if the fd is not in combined_map, we should unregister it. otherwise, it will be polled every
|
3809
|
+
# time, which may cause 100% cpu usage
|
3810
|
+
log.debug('unexpected read event from fd %r', fd)
|
3811
|
+
try:
|
3812
|
+
self._context.poller.unregister_readable(fd)
|
3813
|
+
except Exception: # noqa
|
3814
|
+
pass
|
3595
3815
|
|
3596
|
-
|
3597
|
-
|
3598
|
-
|
3599
|
-
|
3600
|
-
|
3601
|
-
|
3602
|
-
|
3603
|
-
self._context.poller.unregister_writable(fd)
|
3604
|
-
except ExitNow:
|
3605
|
-
raise
|
3606
|
-
except Exception: # noqa
|
3607
|
-
combined_map[fd].handle_error()
|
3608
|
-
else:
|
3609
|
-
log.debug('unexpected write event from fd %r', fd)
|
3610
|
-
try:
|
3816
|
+
for fd in w:
|
3817
|
+
if fd in combined_map:
|
3818
|
+
try:
|
3819
|
+
dispatcher = combined_map[fd]
|
3820
|
+
log.debug('write event caused by %r', dispatcher)
|
3821
|
+
dispatcher.handle_write_event()
|
3822
|
+
if not dispatcher.writable():
|
3611
3823
|
self._context.poller.unregister_writable(fd)
|
3612
|
-
|
3613
|
-
|
3614
|
-
|
3615
|
-
|
3616
|
-
|
3824
|
+
except ExitNow:
|
3825
|
+
raise
|
3826
|
+
except Exception: # noqa
|
3827
|
+
combined_map[fd].handle_error()
|
3828
|
+
else:
|
3829
|
+
log.debug('unexpected write event from fd %r', fd)
|
3830
|
+
try:
|
3831
|
+
self._context.poller.unregister_writable(fd)
|
3832
|
+
except Exception: # noqa
|
3833
|
+
pass
|
3617
3834
|
|
3618
|
-
|
3619
|
-
|
3620
|
-
self._tick()
|
3835
|
+
for group in pgroups:
|
3836
|
+
group.transition()
|
3621
3837
|
|
3622
|
-
|
3623
|
-
|
3838
|
+
self._reap()
|
3839
|
+
self._handle_signal()
|
3840
|
+
self._tick()
|
3624
3841
|
|
3625
|
-
|
3626
|
-
|
3842
|
+
if self._context.state < SupervisorStates.RUNNING:
|
3843
|
+
self._ordered_stop_groups_phase_2()
|
3627
3844
|
|
3628
3845
|
def _tick(self, now: ta.Optional[float] = None) -> None:
|
3629
3846
|
"""Send one or more 'tick' events when the timeslice related to the period for the event type rolls over"""
|
@@ -3694,49 +3911,56 @@ class Supervisor:
|
|
3694
3911
|
log.debug('received %s indicating nothing', signame(sig))
|
3695
3912
|
|
3696
3913
|
|
3697
|
-
|
3698
|
-
|
3914
|
+
########################################
|
3915
|
+
# main.py
|
3699
3916
|
|
3700
3917
|
|
3701
|
-
def main(
|
3918
|
+
def main(
|
3919
|
+
argv: ta.Optional[ta.Sequence[str]] = None,
|
3920
|
+
*,
|
3921
|
+
no_logging: bool = False,
|
3922
|
+
) -> None:
|
3702
3923
|
import argparse
|
3703
3924
|
|
3704
3925
|
parser = argparse.ArgumentParser()
|
3705
3926
|
parser.add_argument('config_file', metavar='config-file')
|
3706
|
-
|
3927
|
+
parser.add_argument('--no-journald', action='store_true')
|
3928
|
+
args = parser.parse_args(argv)
|
3707
3929
|
|
3708
|
-
|
3930
|
+
#
|
3709
3931
|
|
3710
3932
|
if not (cf := args.config_file):
|
3711
3933
|
raise RuntimeError('No config file specified')
|
3712
3934
|
|
3935
|
+
if not no_logging:
|
3936
|
+
configure_standard_logging(
|
3937
|
+
'INFO',
|
3938
|
+
handler_factory=journald_log_handler_factory if not args.no_journald else None,
|
3939
|
+
)
|
3940
|
+
|
3941
|
+
#
|
3942
|
+
|
3713
3943
|
# if we hup, restart by making a new Supervisor()
|
3714
|
-
|
3715
|
-
while True:
|
3944
|
+
for epoch in itertools.count():
|
3716
3945
|
with open(cf) as f:
|
3717
3946
|
config_src = f.read()
|
3947
|
+
|
3718
3948
|
config_dct = json.loads(config_src)
|
3719
3949
|
config: ServerConfig = unmarshal_obj(config_dct, ServerConfig)
|
3720
3950
|
|
3721
3951
|
context = ServerContext(
|
3722
3952
|
config,
|
3953
|
+
epoch=epoch,
|
3723
3954
|
)
|
3724
3955
|
|
3725
|
-
|
3726
|
-
|
3727
|
-
|
3728
|
-
|
3729
|
-
|
3730
|
-
if test or (context.state < SupervisorStates.RESTARTING):
|
3731
|
-
break
|
3732
|
-
|
3956
|
+
supervisor = Supervisor(context)
|
3957
|
+
try:
|
3958
|
+
supervisor.main()
|
3959
|
+
except ExitNow:
|
3960
|
+
pass
|
3733
3961
|
|
3734
|
-
|
3735
|
-
|
3736
|
-
try:
|
3737
|
-
d.main()
|
3738
|
-
except ExitNow:
|
3739
|
-
pass
|
3962
|
+
if context.state < SupervisorStates.RESTARTING:
|
3963
|
+
break
|
3740
3964
|
|
3741
3965
|
|
3742
3966
|
if __name__ == '__main__':
|