ominfra 0.0.0.dev156__py3-none-any.whl → 0.0.0.dev158__py3-none-any.whl

ominfra/scripts/supervisor.py

@@ -142,9 +142,9 @@ InjectorBindingOrBindings = ta.Union['InjectorBinding', 'InjectorBindings']
 ConfigMapping = ta.Mapping[str, ta.Any]
 
 # ../../omlish/http/handlers.py
-HttpHandler = ta.Callable[['HttpHandlerRequest'], 'HttpHandlerResponse']
+HttpHandler = ta.Callable[['HttpHandlerRequest'], 'HttpHandlerResponse']  # ta.TypeAlias
 
-# ../../omlish/http/coroserver.py
+# ../../omlish/http/coro/server.py
 CoroHttpServerFactory = ta.Callable[[SocketAddress], 'CoroHttpServer']
 
 
@@ -1697,8 +1697,8 @@ class _CachedNullary(_AbstractCachedNullary):
         return self._value
 
 
-def cached_nullary(fn):  # ta.Callable[..., T]) -> ta.Callable[..., T]:
-    return _CachedNullary(fn)
+def cached_nullary(fn: CallableT) -> CallableT:
+    return _CachedNullary(fn)  # type: ignore
 
 
 def static_init(fn: CallableT) -> CallableT:
@@ -2202,6 +2202,13 @@ json_dump_compact: ta.Callable[..., bytes] = functools.partial(json.dump, **JSON
 json_dumps_compact: ta.Callable[..., str] = functools.partial(json.dumps, **JSON_COMPACT_KWARGS)
 
 
+########################################
+# ../../../omlish/lite/logs.py
+
+
+log = logging.getLogger(__name__)
+
+
 ########################################
 # ../../../omlish/lite/maybes.py
 
@@ -2486,6 +2493,116 @@ class Func3(ta.Generic[A0, A1, A2, T]):
         return self.fn(a0, a1, a2)
 
 
+########################################
+# ../../../omlish/logs/filters.py
+
+
+class TidLogFilter(logging.Filter):
+    def filter(self, record):
+        record.tid = threading.get_native_id()
+        return True
+
+
+########################################
+# ../../../omlish/logs/proxy.py
+
+
+class ProxyLogFilterer(logging.Filterer):
+    def __init__(self, underlying: logging.Filterer) -> None:  # noqa
+        self._underlying = underlying
+
+    @property
+    def underlying(self) -> logging.Filterer:
+        return self._underlying
+
+    @property
+    def filters(self):
+        return self._underlying.filters
+
+    @filters.setter
+    def filters(self, filters):
+        self._underlying.filters = filters
+
+    def addFilter(self, filter):  # noqa
+        self._underlying.addFilter(filter)
+
+    def removeFilter(self, filter):  # noqa
+        self._underlying.removeFilter(filter)
+
+    def filter(self, record):
+        return self._underlying.filter(record)
+
+
+class ProxyLogHandler(ProxyLogFilterer, logging.Handler):
+    def __init__(self, underlying: logging.Handler) -> None:  # noqa
+        ProxyLogFilterer.__init__(self, underlying)
+
+    _underlying: logging.Handler
+
+    @property
+    def underlying(self) -> logging.Handler:
+        return self._underlying
+
+    def get_name(self):
+        return self._underlying.get_name()
+
+    def set_name(self, name):
+        self._underlying.set_name(name)
+
+    @property
+    def name(self):
+        return self._underlying.name
+
+    @property
+    def level(self):
+        return self._underlying.level
+
+    @level.setter
+    def level(self, level):
+        self._underlying.level = level
+
+    @property
+    def formatter(self):
+        return self._underlying.formatter
+
+    @formatter.setter
+    def formatter(self, formatter):
+        self._underlying.formatter = formatter
+
+    def createLock(self):
+        self._underlying.createLock()
+
+    def acquire(self):
+        self._underlying.acquire()
+
+    def release(self):
+        self._underlying.release()
+
+    def setLevel(self, level):
+        self._underlying.setLevel(level)
+
+    def format(self, record):
+        return self._underlying.format(record)
+
+    def emit(self, record):
+        self._underlying.emit(record)
+
+    def handle(self, record):
+        return self._underlying.handle(record)
+
+    def setFormatter(self, fmt):
+        self._underlying.setFormatter(fmt)
+
+    def flush(self):
+        self._underlying.flush()
+
+    def close(self):
+        self._underlying.close()
+
+    def handleError(self, record):
+        self._underlying.handleError(record)
+
+
 ########################################
 # ../events.py
 
@@ -2858,6 +2975,44 @@ def decode_wait_status(sts: int) -> ta.Tuple[Rc, str]:
         return Rc(-1), msg
 
 
+##
+
+
+class WaitedPid(ta.NamedTuple):
+    pid: Pid
+    sts: Rc
+
+
+def waitpid(
+        *,
+        log: ta.Optional[logging.Logger] = None,
+) -> ta.Optional[WaitedPid]:
+    # Need pthread_sigmask here to avoid concurrent sigchld, but Python doesn't offer in Python < 3.4. There is
+    # still a race condition here; we can get a sigchld while we're sitting in the waitpid call. However, AFAICT, if
+    # waitpid is interrupted by SIGCHLD, as long as we call waitpid again (which happens every so often during the
+    # normal course in the mainloop), we'll eventually reap the child that we tried to reap during the interrupted
+    # call. At least on Linux, this appears to be true, or at least stopping 50 processes at once never left zombies
+    # lying around.
+    try:
+        pid, sts = os.waitpid(-1, os.WNOHANG)
+
+    except OSError as exc:
+        code = exc.args[0]
+
+        if code not in (errno.ECHILD, errno.EINTR):
+            if log is not None:
+                log.critical('waitpid error %r; a process may not be cleaned up properly', code)
+
+        if code == errno.EINTR:
+            if log is not None:
+                log.debug('EINTR during reap')
+
+        return None
+
+    else:
+        return WaitedPid(pid, sts)  # type: ignore
+
+
 ########################################
 # ../utils/users.py
 
@@ -4635,333 +4790,63 @@ class Injection:
             tag: ta.Any = None,
             array: ta.Optional[bool] = None,  # noqa
 
-            to_fn: ta.Any = None,
-            to_ctor: ta.Any = None,
-            to_const: ta.Any = None,
-            to_key: ta.Any = None,
-
-            singleton: bool = False,
-
-            eager: bool = False,
-    ) -> InjectorBindingOrBindings:
-        return InjectorBinder.bind(
-            obj,
-
-            key=key,
-            tag=tag,
-            array=array,
-
-            to_fn=to_fn,
-            to_ctor=to_ctor,
-            to_const=to_const,
-            to_key=to_key,
-
-            singleton=singleton,
-
-            eager=eager,
-        )
-
-    # helpers
-
-    @classmethod
-    def bind_factory(
-            cls,
-            fn: ta.Callable[..., T],
-            cls_: U,
-            ann: ta.Any = None,
-    ) -> InjectorBindingOrBindings:
-        return cls.bind(make_injector_factory(fn, cls_, ann))
-
-    @classmethod
-    def bind_array(
-            cls,
-            obj: ta.Any = None,
-            *,
-            tag: ta.Any = None,
-    ) -> InjectorBindingOrBindings:
-        return bind_injector_array(obj, tag=tag)
-
-    @classmethod
-    def bind_array_type(
-            cls,
-            ele: ta.Union[InjectorKey, InjectorKeyCls],
-            cls_: U,
-            ann: ta.Any = None,
-    ) -> InjectorBindingOrBindings:
-        return cls.bind(make_injector_array_type(ele, cls_, ann))
-
-
-inj = Injection
-
-
-########################################
-# ../../../omlish/lite/logs.py
-"""
-TODO:
- - translate json keys
- - debug
-"""
-
-
-log = logging.getLogger(__name__)
-
-
-##
-
-
-class TidLogFilter(logging.Filter):
-
-    def filter(self, record):
-        record.tid = threading.get_native_id()
-        return True
-
-
-##
-
-
-class JsonLogFormatter(logging.Formatter):
-
-    KEYS: ta.Mapping[str, bool] = {
-        'name': False,
-        'msg': False,
-        'args': False,
-        'levelname': False,
-        'levelno': False,
-        'pathname': False,
-        'filename': False,
-        'module': False,
-        'exc_info': True,
-        'exc_text': True,
-        'stack_info': True,
-        'lineno': False,
-        'funcName': False,
-        'created': False,
-        'msecs': False,
-        'relativeCreated': False,
-        'thread': False,
-        'threadName': False,
-        'processName': False,
-        'process': False,
-    }
-
-    def format(self, record: logging.LogRecord) -> str:
-        dct = {
-            k: v
-            for k, o in self.KEYS.items()
-            for v in [getattr(record, k)]
-            if not (o and v is None)
-        }
-        return json_dumps_compact(dct)
-
-
-##
-
-
-STANDARD_LOG_FORMAT_PARTS = [
-    ('asctime', '%(asctime)-15s'),
-    ('process', 'pid=%(process)-6s'),
-    ('thread', 'tid=%(thread)x'),
-    ('levelname', '%(levelname)s'),
-    ('name', '%(name)s'),
-    ('separator', '::'),
-    ('message', '%(message)s'),
-]
-
-
-class StandardLogFormatter(logging.Formatter):
-
-    @staticmethod
-    def build_log_format(parts: ta.Iterable[ta.Tuple[str, str]]) -> str:
-        return ' '.join(v for k, v in parts)
-
-    converter = datetime.datetime.fromtimestamp  # type: ignore
-
-    def formatTime(self, record, datefmt=None):
-        ct = self.converter(record.created)  # type: ignore
-        if datefmt:
-            return ct.strftime(datefmt)  # noqa
-        else:
-            t = ct.strftime('%Y-%m-%d %H:%M:%S')
-            return '%s.%03d' % (t, record.msecs)  # noqa
-
-
-##
-
-
-class ProxyLogFilterer(logging.Filterer):
-    def __init__(self, underlying: logging.Filterer) -> None:  # noqa
-        self._underlying = underlying
-
-    @property
-    def underlying(self) -> logging.Filterer:
-        return self._underlying
-
-    @property
-    def filters(self):
-        return self._underlying.filters
-
-    @filters.setter
-    def filters(self, filters):
-        self._underlying.filters = filters
-
-    def addFilter(self, filter):  # noqa
-        self._underlying.addFilter(filter)
-
-    def removeFilter(self, filter):  # noqa
-        self._underlying.removeFilter(filter)
-
-    def filter(self, record):
-        return self._underlying.filter(record)
-
-
-class ProxyLogHandler(ProxyLogFilterer, logging.Handler):
-    def __init__(self, underlying: logging.Handler) -> None:  # noqa
-        ProxyLogFilterer.__init__(self, underlying)
-
-    _underlying: logging.Handler
-
-    @property
-    def underlying(self) -> logging.Handler:
-        return self._underlying
-
-    def get_name(self):
-        return self._underlying.get_name()
-
-    def set_name(self, name):
-        self._underlying.set_name(name)
-
-    @property
-    def name(self):
-        return self._underlying.name
-
-    @property
-    def level(self):
-        return self._underlying.level
-
-    @level.setter
-    def level(self, level):
-        self._underlying.level = level
-
-    @property
-    def formatter(self):
-        return self._underlying.formatter
-
-    @formatter.setter
-    def formatter(self, formatter):
-        self._underlying.formatter = formatter
-
-    def createLock(self):
-        self._underlying.createLock()
-
-    def acquire(self):
-        self._underlying.acquire()
-
-    def release(self):
-        self._underlying.release()
-
-    def setLevel(self, level):
-        self._underlying.setLevel(level)
-
-    def format(self, record):
-        return self._underlying.format(record)
-
-    def emit(self, record):
-        self._underlying.emit(record)
-
-    def handle(self, record):
-        return self._underlying.handle(record)
-
-    def setFormatter(self, fmt):
-        self._underlying.setFormatter(fmt)
-
-    def flush(self):
-        self._underlying.flush()
-
-    def close(self):
-        self._underlying.close()
-
-    def handleError(self, record):
-        self._underlying.handleError(record)
-
-
-##
-
-
-class StandardLogHandler(ProxyLogHandler):
-    pass
-
-
-##
-
-
-@contextlib.contextmanager
-def _locking_logging_module_lock() -> ta.Iterator[None]:
-    if hasattr(logging, '_acquireLock'):
-        logging._acquireLock()  # noqa
-        try:
-            yield
-        finally:
-            logging._releaseLock()  # type: ignore  # noqa
-
-    elif hasattr(logging, '_lock'):
-        # https://github.com/python/cpython/commit/74723e11109a320e628898817ab449b3dad9ee96
-        with logging._lock:  # noqa
-            yield
-
-    else:
-        raise Exception("Can't find lock in logging module")
-
-
-def configure_standard_logging(
-        level: ta.Union[int, str] = logging.INFO,
-        *,
-        json: bool = False,
-        target: ta.Optional[logging.Logger] = None,
-        force: bool = False,
-        handler_factory: ta.Optional[ta.Callable[[], logging.Handler]] = None,
-) -> ta.Optional[StandardLogHandler]:
-    with _locking_logging_module_lock():
-        if target is None:
-            target = logging.root
-
-        #
-
-        if not force:
-            if any(isinstance(h, StandardLogHandler) for h in list(target.handlers)):
-                return None
-
-        #
+            to_fn: ta.Any = None,
+            to_ctor: ta.Any = None,
+            to_const: ta.Any = None,
+            to_key: ta.Any = None,
 
-        if handler_factory is not None:
-            handler = handler_factory()
-        else:
-            handler = logging.StreamHandler()
+            singleton: bool = False,
 
-        #
+            eager: bool = False,
+    ) -> InjectorBindingOrBindings:
+        return InjectorBinder.bind(
+            obj,
 
-        formatter: logging.Formatter
-        if json:
-            formatter = JsonLogFormatter()
-        else:
-            formatter = StandardLogFormatter(StandardLogFormatter.build_log_format(STANDARD_LOG_FORMAT_PARTS))
-        handler.setFormatter(formatter)
+            key=key,
+            tag=tag,
+            array=array,
 
-        #
+            to_fn=to_fn,
+            to_ctor=to_ctor,
+            to_const=to_const,
+            to_key=to_key,
 
-        handler.addFilter(TidLogFilter())
+            singleton=singleton,
 
-        #
+            eager=eager,
+        )
 
-        target.addHandler(handler)
+    # helpers
 
-        #
+    @classmethod
+    def bind_factory(
+            cls,
+            fn: ta.Callable[..., T],
+            cls_: U,
+            ann: ta.Any = None,
+    ) -> InjectorBindingOrBindings:
+        return cls.bind(make_injector_factory(fn, cls_, ann))
 
-        if level is not None:
-            target.setLevel(level)
+    @classmethod
+    def bind_array(
+            cls,
+            obj: ta.Any = None,
+            *,
+            tag: ta.Any = None,
+    ) -> InjectorBindingOrBindings:
+        return bind_injector_array(obj, tag=tag)
 
-        #
+    @classmethod
+    def bind_array_type(
+            cls,
+            ele: ta.Union[InjectorKey, InjectorKeyCls],
+            cls_: U,
+            ann: ta.Any = None,
+    ) -> InjectorBindingOrBindings:
+        return cls.bind(make_injector_array_type(ele, cls_, ann))
 
-        return StandardLogHandler(handler)
+
+inj = Injection
 
 
 ########################################
@@ -4971,6 +4856,7 @@ TODO:
  - pickle stdlib objs? have to pin to 3.8 pickle protocol, will be cross-version
  - namedtuple
  - literals
+ - newtypes?
 """
 
 
@@ -5425,6 +5311,60 @@ def check_runtime_version() -> None:
         raise OSError(f'Requires python {REQUIRED_PYTHON_VERSION}, got {sys.version_info} from {sys.executable}')  # noqa
 
 
+########################################
+# ../../../omlish/logs/json.py
+"""
+TODO:
+ - translate json keys
+"""
+
+
+class JsonLogFormatter(logging.Formatter):
+    KEYS: ta.Mapping[str, bool] = {
+        'name': False,
+        'msg': False,
+        'args': False,
+        'levelname': False,
+        'levelno': False,
+        'pathname': False,
+        'filename': False,
+        'module': False,
+        'exc_info': True,
+        'exc_text': True,
+        'stack_info': True,
+        'lineno': False,
+        'funcName': False,
+        'created': False,
+        'msecs': False,
+        'relativeCreated': False,
+        'thread': False,
+        'threadName': False,
+        'processName': False,
+        'process': False,
+    }
+
+    def __init__(
+            self,
+            *args: ta.Any,
+            json_dumps: ta.Optional[ta.Callable[[ta.Any], str]] = None,
+            **kwargs: ta.Any,
+    ) -> None:
+        super().__init__(*args, **kwargs)
+
+        if json_dumps is None:
+            json_dumps = json_dumps_compact
+        self._json_dumps = json_dumps
+
+    def format(self, record: logging.LogRecord) -> str:
+        dct = {
+            k: v
+            for k, o in self.KEYS.items()
+            for v in [getattr(record, k)]
+            if not (o and v is None)
+        }
+        return self._json_dumps(dct)
+
+
 ########################################
 # ../../../omlish/os/journald.py
 
@@ -5813,6 +5753,126 @@ class UnsupportedMethodHttpHandlerError(Exception):
     pass
 
 
+########################################
+# ../../../omlish/logs/standard.py
+"""
+TODO:
+ - structured
+ - prefixed
+ - debug
+"""
+
+
+##
+
+
+STANDARD_LOG_FORMAT_PARTS = [
+    ('asctime', '%(asctime)-15s'),
+    ('process', 'pid=%(process)-6s'),
+    ('thread', 'tid=%(thread)x'),
+    ('levelname', '%(levelname)s'),
+    ('name', '%(name)s'),
+    ('separator', '::'),
+    ('message', '%(message)s'),
+]
+
+
+class StandardLogFormatter(logging.Formatter):
+    @staticmethod
+    def build_log_format(parts: ta.Iterable[ta.Tuple[str, str]]) -> str:
+        return ' '.join(v for k, v in parts)
+
+    converter = datetime.datetime.fromtimestamp  # type: ignore
+
+    def formatTime(self, record, datefmt=None):
+        ct = self.converter(record.created)  # type: ignore
+        if datefmt:
+            return ct.strftime(datefmt)  # noqa
+        else:
+            t = ct.strftime('%Y-%m-%d %H:%M:%S')
+            return '%s.%03d' % (t, record.msecs)  # noqa
+
+
+##
+
+
+class StandardLogHandler(ProxyLogHandler):
+    pass
+
+
+##
+
+
+@contextlib.contextmanager
+def _locking_logging_module_lock() -> ta.Iterator[None]:
+    if hasattr(logging, '_acquireLock'):
+        logging._acquireLock()  # noqa
+        try:
+            yield
+        finally:
+            logging._releaseLock()  # type: ignore  # noqa
+
+    elif hasattr(logging, '_lock'):
+        # https://github.com/python/cpython/commit/74723e11109a320e628898817ab449b3dad9ee96
+        with logging._lock:  # noqa
+            yield
+
+    else:
+        raise Exception("Can't find lock in logging module")
+
+
+def configure_standard_logging(
+        level: ta.Union[int, str] = logging.INFO,
+        *,
+        json: bool = False,
+        target: ta.Optional[logging.Logger] = None,
+        force: bool = False,
+        handler_factory: ta.Optional[ta.Callable[[], logging.Handler]] = None,
+) -> ta.Optional[StandardLogHandler]:
+    with _locking_logging_module_lock():
+        if target is None:
+            target = logging.root
+
+        #
+
+        if not force:
+            if any(isinstance(h, StandardLogHandler) for h in list(target.handlers)):
+                return None
+
+        #
+
+        if handler_factory is not None:
+            handler = handler_factory()
+        else:
+            handler = logging.StreamHandler()
+
+        #
+
+        formatter: logging.Formatter
+        if json:
+            formatter = JsonLogFormatter()
+        else:
+            formatter = StandardLogFormatter(StandardLogFormatter.build_log_format(STANDARD_LOG_FORMAT_PARTS))
+        handler.setFormatter(formatter)
+
+        #
+
+        handler.addFilter(TidLogFilter())
+
+        #
+
+        target.addHandler(handler)
+
+        #
+
+        if level is not None:
+            target.setLevel(level)
+
+        #
+
+        return StandardLogHandler(handler)
+
+
 ########################################
 # ../configs.py
 
@@ -6049,6 +6109,8 @@ class ServerConfig:
 
     groups: ta.Optional[ta.Sequence[ProcessGroupConfig]] = None
 
+    group_config_dirs: ta.Optional[ta.Sequence[str]] = None
+
     @classmethod
     def new(
             cls,
@@ -6104,7 +6166,7 @@ def parse_logging_level(value: ta.Union[str, int]) -> int:
 
 
 ########################################
-# ../../../omlish/http/coroserver.py
+# ../../../omlish/http/coro/server.py
 # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
 # --------------------------------------------
 #
@@ -6844,7 +6906,7 @@ class ProcessGroup(
 
 
 ########################################
-# ../../../omlish/io/fdio/corohttp.py
+# ../../../omlish/http/coro/fdio.py
 
 
 class CoroHttpServerConnectionFdioHandler(SocketFdioHandler):
@@ -9054,7 +9116,7 @@ class Supervisor:
         if depth >= 100:
             return
 
-        wp = waitpid()
+        wp = waitpid(log=log)
 
         if wp is None or not wp.pid:
             return
@@ -9095,34 +9157,6 @@ class Supervisor:
                 self._event_callbacks.notify(event(this_tick, self))
 
 
-##
-
-
-class WaitedPid(ta.NamedTuple):
-    pid: Pid
-    sts: Rc
-
-
-def waitpid() -> ta.Optional[WaitedPid]:
-    # Need pthread_sigmask here to avoid concurrent sigchld, but Python doesn't offer in Python < 3.4. There is
-    # still a race condition here; we can get a sigchld while we're sitting in the waitpid call. However, AFAICT, if
-    # waitpid is interrupted by SIGCHLD, as long as we call waitpid again (which happens every so often during the
-    # normal course in the mainloop), we'll eventually reap the child that we tried to reap during the interrupted
-    # call. At least on Linux, this appears to be true, or at least stopping 50 processes at once never left zombies
-    # lying around.
-    try:
-        pid, sts = os.waitpid(-1, os.WNOHANG)
-    except OSError as exc:
-        code = exc.args[0]
-        if code not in (errno.ECHILD, errno.EINTR):
-            log.critical('waitpid error %r; a process may not be cleaned up properly', code)
-        if code == errno.EINTR:
-            log.debug('EINTR during reap')
-        return None
-    else:
-        return WaitedPid(pid, sts)  # type: ignore
-
-
 ########################################
 # ../inject.py