omlish 0.0.0.dev156__tar.gz → 0.0.0.dev158__tar.gz

{omlish-0.0.0.dev156/omlish.egg-info → omlish-0.0.0.dev158}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omlish
-Version: 0.0.0.dev156
+Version: 0.0.0.dev158
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause

{omlish-0.0.0.dev156 → omlish-0.0.0.dev158}/omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev156'
-__revision__ = '594c80ed50a0c5515f35cac4b31a4ed482993a4b'
+__version__ = '0.0.0.dev158'
+__revision__ = 'af6daf87e05e81846b224e4d0031cd101a27158b'
 
 
 #

{omlish-0.0.0.dev156 → omlish-0.0.0.dev158}/omlish/argparse/cli.py

@@ -14,6 +14,8 @@ import sys
 import typing as ta
 
 from ..lite.check import check
+from ..lite.reflect import get_optional_alias_arg
+from ..lite.reflect import is_optional_alias
 
 
 T = ta.TypeVar('T')
@@ -120,6 +122,8 @@ def _get_argparse_arg_ann_kwargs(ann: ta.Any) -> ta.Mapping[str, ta.Any]:
         return {'action': 'store_true'}
     elif ann is list:
         return {'action': 'append'}
+    elif is_optional_alias(ann):
+        return _get_argparse_arg_ann_kwargs(get_optional_alias_arg(ann))
     else:
         raise TypeError(ann)
 

{omlish-0.0.0.dev156/omlish/lite → omlish-0.0.0.dev158/omlish/asyncs}/asyncio/subprocesses.py

@@ -1,17 +1,18 @@
 # ruff: noqa: UP006 UP007
+# @omlish-lite
 import asyncio.base_subprocess
 import asyncio.subprocess
 import contextlib
 import dataclasses as dc
 import functools
+import logging
 import subprocess
 import sys
 import typing as ta
 
-from ...asyncs.asyncio.timeouts import asyncio_maybe_timeout
-from ..check import check
-from ..logs import log
-from ..subprocesses import AbstractSubprocesses
+from ...lite.check import check
+from ...subprocesses import AbstractAsyncSubprocesses
+from .timeouts import asyncio_maybe_timeout
 
 
 T = ta.TypeVar('T')
@@ -25,6 +26,8 @@ class AsyncioProcessCommunicator:
             self,
             proc: asyncio.subprocess.Process,
             loop: ta.Optional[ta.Any] = None,
+            *,
+            log: ta.Optional[logging.Logger] = None,
     ) -> None:
         super().__init__()
 
@@ -33,6 +36,7 @@ class AsyncioProcessCommunicator:
 
         self._proc = proc
         self._loop = loop
+        self._log = log
 
         self._transport: asyncio.base_subprocess.BaseSubprocessTransport = check.isinstance(
             proc._transport,  # type: ignore  # noqa
@@ -48,19 +52,19 @@ class AsyncioProcessCommunicator:
         try:
             if input is not None:
                 stdin.write(input)
-                if self._debug:
-                    log.debug('%r communicate: feed stdin (%s bytes)', self, len(input))
+                if self._debug and self._log is not None:
+                    self._log.debug('%r communicate: feed stdin (%s bytes)', self, len(input))
 
             await stdin.drain()
 
         except (BrokenPipeError, ConnectionResetError) as exc:
             # communicate() ignores BrokenPipeError and ConnectionResetError. write() and drain() can raise these
             # exceptions.
-            if self._debug:
-                log.debug('%r communicate: stdin got %r', self, exc)
+            if self._debug and self._log is not None:
+                self._log.debug('%r communicate: stdin got %r', self, exc)
 
-        if self._debug:
-            log.debug('%r communicate: close stdin', self)
+        if self._debug and self._log is not None:
+            self._log.debug('%r communicate: close stdin', self)
 
         stdin.close()
 
@@ -76,15 +80,15 @@ class AsyncioProcessCommunicator:
             check.equal(fd, 1)
             stream = check.not_none(self._proc.stdout)
 
-        if self._debug:
+        if self._debug and self._log is not None:
             name = 'stdout' if fd == 1 else 'stderr'
-            log.debug('%r communicate: read %s', self, name)
+            self._log.debug('%r communicate: read %s', self, name)
 
         output = await stream.read()
 
-        if self._debug:
+        if self._debug and self._log is not None:
             name = 'stdout' if fd == 1 else 'stderr'
-            log.debug('%r communicate: close %s', self, name)
+            self._log.debug('%r communicate: close %s', self, name)
 
         transport.close()
 
@@ -133,7 +137,7 @@ class AsyncioProcessCommunicator:
 ##
 
 
-class AsyncioSubprocesses(AbstractSubprocesses):
+class AsyncioSubprocesses(AbstractAsyncSubprocesses):
     async def communicate(
             self,
             proc: asyncio.subprocess.Process,
@@ -230,44 +234,5 @@ class AsyncioSubprocesses(AbstractSubprocesses):
         with self.prepare_and_wrap(*cmd, stdout=subprocess.PIPE, check=True, **kwargs) as (cmd, kwargs):  # noqa
             return check.not_none((await self.run(*cmd, **kwargs)).stdout)
 
-    async def check_output_str(
-            self,
-            *cmd: str,
-            **kwargs: ta.Any,
-    ) -> str:
-        return (await self.check_output(*cmd, **kwargs)).decode().strip()
-
-    #
-
-    async def try_call(
-            self,
-            *cmd: str,
-            **kwargs: ta.Any,
-    ) -> bool:
-        if isinstance(await self.async_try_fn(self.check_call, *cmd, **kwargs), Exception):
-            return False
-        else:
-            return True
-
-    async def try_output(
-            self,
-            *cmd: str,
-            **kwargs: ta.Any,
-    ) -> ta.Optional[bytes]:
-        if isinstance(ret := await self.async_try_fn(self.check_output, *cmd, **kwargs), Exception):
-            return None
-        else:
-            return ret
-
-    async def try_output_str(
-            self,
-            *cmd: str,
-            **kwargs: ta.Any,
-    ) -> ta.Optional[str]:
-        if (ret := await self.try_output(*cmd, **kwargs)) is None:
-            return None
-        else:
-            return ret.decode().strip()
-
 
 asyncio_subprocesses = AsyncioSubprocesses()

{omlish-0.0.0.dev156 → omlish-0.0.0.dev158}/omlish/bootstrap/sys.py

@@ -21,8 +21,8 @@ from .base import SimpleBootstrap
 
 if ta.TYPE_CHECKING:
     from .. import libc
-    from .. import logs
     from ..formats import dotenv
+    from ..logs import all as logs
     from ..os import pidfile
 
 else:

omlish-0.0.0.dev156/omlish/io/fdio/corohttp.py → omlish-0.0.0.dev158/omlish/http/coro/fdio.py

@@ -1,14 +1,15 @@
 # ruff: noqa: UP006 UP007
+# @omlish-lite
 import socket
 import typing as ta
 
-from ...http.coroserver import CoroHttpServer
-from ...http.handlers import HttpHandler
+from ...io.buffers import IncrementalWriteBuffer
+from ...io.buffers import ReadableListBuffer
+from ...io.fdio.handlers import SocketFdioHandler
 from ...lite.check import check
 from ...lite.socket import SocketAddress
-from ..buffers import IncrementalWriteBuffer
-from ..buffers import ReadableListBuffer
-from .handlers import SocketFdioHandler
+from ..handlers import HttpHandler
+from .server import CoroHttpServer
 
 
 class CoroHttpServerConnectionFdioHandler(SocketFdioHandler):

omlish-0.0.0.dev156/omlish/http/coroserver.py → omlish-0.0.0.dev158/omlish/http/coro/server.py

@@ -63,18 +63,18 @@ import textwrap
 import time
 import typing as ta
 
-from ..lite.check import check
-from ..lite.socket import SocketAddress
-from ..lite.socket import SocketHandler
-from .handlers import HttpHandler
-from .handlers import HttpHandlerRequest
-from .handlers import UnsupportedMethodHttpHandlerError
-from .parsing import EmptyParsedHttpResult
-from .parsing import HttpRequestParser
-from .parsing import ParsedHttpRequest
-from .parsing import ParseHttpRequestError
-from .versions import HttpProtocolVersion
-from .versions import HttpProtocolVersions
+from ...lite.check import check
+from ...lite.socket import SocketAddress
+from ...lite.socket import SocketHandler
+from ..handlers import HttpHandler
+from ..handlers import HttpHandlerRequest
+from ..handlers import UnsupportedMethodHttpHandlerError
+from ..parsing import EmptyParsedHttpResult
+from ..parsing import HttpRequestParser
+from ..parsing import ParsedHttpRequest
+from ..parsing import ParseHttpRequestError
+from ..versions import HttpProtocolVersion
+from ..versions import HttpProtocolVersions
 
 
 CoroHttpServerFactory = ta.Callable[[SocketAddress], 'CoroHttpServer']

{omlish-0.0.0.dev156 → omlish-0.0.0.dev158}/omlish/http/handlers.py

@@ -8,7 +8,7 @@ from ..lite.socket import SocketAddress
 from .parsing import HttpHeaders
 
 
-HttpHandler = ta.Callable[['HttpHandlerRequest'], 'HttpHandlerResponse']
+HttpHandler = ta.Callable[['HttpHandlerRequest'], 'HttpHandlerResponse']  # ta.TypeAlias
 
 
 @dc.dataclass(frozen=True)

{omlish-0.0.0.dev156 → omlish-0.0.0.dev158}/omlish/lite/cached.py

@@ -35,8 +35,8 @@ class _CachedNullary(_AbstractCachedNullary):
         return self._value
 
 
-def cached_nullary(fn):  # ta.Callable[..., T]) -> ta.Callable[..., T]:
-    return _CachedNullary(fn)
+def cached_nullary(fn: CallableT) -> CallableT:
+    return _CachedNullary(fn)  # type: ignore
 
 
 def static_init(fn: CallableT) -> CallableT:

omlish-0.0.0.dev158/omlish/lite/logs.py

@@ -0,0 +1,4 @@
+import logging
+
+
+log = logging.getLogger(__name__)

{omlish-0.0.0.dev156 → omlish-0.0.0.dev158}/omlish/lite/marshal.py

@@ -3,6 +3,7 @@ TODO:
  - pickle stdlib objs? have to pin to 3.8 pickle protocol, will be cross-version
  - namedtuple
  - literals
+ - newtypes?
 """
 # ruff: noqa: UP006 UP007
 import abc

omlish-0.0.0.dev156/omlish/logs/__init__.py → omlish-0.0.0.dev158/omlish/logs/all.py

@@ -1,32 +1,37 @@
-from .configs import (  # noqa
-    configure_standard_logging,
+from .color import (  # noqa
+    ColorLogFormatter,
 )
 
-from .formatters import (  # noqa
-    ColorLogFormatter,
+from .filters import (  # noqa
+    TidLogFilter,
 )
 
 from .handlers import (  # noqa
     ListHandler,
 )
 
-from .utils import (  # noqa
-    error_logging,
+from .json import (  # noqa
+    JsonLogFormatter,
 )
 
+from .noisy import (  # noqa
+    silence_noisy_loggers,
+)
 
-##
-
-
-from ..lite.logs import (  # noqa
-    TidLogFilter,
-    JsonLogFormatter,
+from .proxy import (  # noqa
+    ProxyLogFilterer,
+    ProxyLogHandler,
+)
 
+from .standard import (  # noqa
     STANDARD_LOG_FORMAT_PARTS,
     StandardLogFormatter,
 
-    ProxyLogFilterer,
-    ProxyLogHandler,
-
     StandardLogHandler,
+
+    configure_standard_logging,
+)
+
+from .utils import (  # noqa
+    error_logging,
 )

omlish-0.0.0.dev156/omlish/logs/formatters.py → omlish-0.0.0.dev158/omlish/logs/color.py

@@ -3,11 +3,10 @@ import logging
 import typing as ta
 
 from .. import term
-from ..lite.logs import StandardLogFormatter
+from .standard import StandardLogFormatter
 
 
 class ColorLogFormatter(StandardLogFormatter):
-
     LEVEL_COLORS: ta.Mapping[int, term.SGRs.FG] = {
         logging.WARNING: term.SGRs.FG.BRIGHT_YELLOW,
         logging.ERROR: term.SGRs.FG.BRIGHT_RED,

omlish-0.0.0.dev158/omlish/logs/configs.py

@@ -0,0 +1,26 @@
+"""
+https://docs.python.org/3/howto/logging.html#configuring-logging
+"""
+import dataclasses as dc
+import typing as ta
+
+
+##
+
+
+FilterConfig: ta.TypeAlias = dict[str, ta.Any]
+FormatterConfig: ta.TypeAlias = dict[str, ta.Any]
+HandlerConfig: ta.TypeAlias = dict[str, ta.Any]
+LoggerConfig: ta.TypeAlias = dict[str, ta.Any]
+
+
+@dc.dataclass()
+class DictConfig:
+    version: int = 1
+    incremental: bool = False
+    disable_existing_loggers: bool = False
+    filters: dict[str, FilterConfig] = dc.field(default_factory=dict)
+    formatters: dict[str, FormatterConfig] = dc.field(default_factory=dict)
+    handlers: dict[str, HandlerConfig] = dc.field(default_factory=dict)
+    loggers: dict[str, LoggerConfig] = dc.field(default_factory=dict)
+    root: LoggerConfig | None = None

omlish-0.0.0.dev158/omlish/logs/filters.py

@@ -0,0 +1,10 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import logging
+import threading
+
+
+class TidLogFilter(logging.Filter):
+    def filter(self, record):
+        record.tid = threading.get_native_id()
+        return True

{omlish-0.0.0.dev156 → omlish-0.0.0.dev158}/omlish/logs/handlers.py

@@ -1,10 +1,13 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
 import logging
+import typing as ta
 
 
 class ListHandler(logging.Handler):
     def __init__(self) -> None:
         super().__init__()
-        self.records: list[logging.LogRecord] = []
+        self.records: ta.List[logging.LogRecord] = []
 
     def emit(self, record: logging.LogRecord) -> None:
         self.records.append(record)

omlish-0.0.0.dev158/omlish/logs/json.py

@@ -0,0 +1,56 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+"""
+TODO:
+ - translate json keys
+"""
+import logging
+import typing as ta
+
+from ..lite.json import json_dumps_compact
+
+
+class JsonLogFormatter(logging.Formatter):
+    KEYS: ta.Mapping[str, bool] = {
+        'name': False,
+        'msg': False,
+        'args': False,
+        'levelname': False,
+        'levelno': False,
+        'pathname': False,
+        'filename': False,
+        'module': False,
+        'exc_info': True,
+        'exc_text': True,
+        'stack_info': True,
+        'lineno': False,
+        'funcName': False,
+        'created': False,
+        'msecs': False,
+        'relativeCreated': False,
+        'thread': False,
+        'threadName': False,
+        'processName': False,
+        'process': False,
+    }
+
+    def __init__(
+            self,
+            *args: ta.Any,
+            json_dumps: ta.Optional[ta.Callable[[ta.Any], str]] = None,
+            **kwargs: ta.Any,
+    ) -> None:
+        super().__init__(*args, **kwargs)
+
+        if json_dumps is None:
+            json_dumps = json_dumps_compact
+        self._json_dumps = json_dumps
+
+    def format(self, record: logging.LogRecord) -> str:
+        dct = {
+            k: v
+            for k, o in self.KEYS.items()
+            for v in [getattr(record, k)]
+            if not (o and v is None)
+        }
+        return self._json_dumps(dct)

omlish-0.0.0.dev158/omlish/logs/proxy.py

@@ -0,0 +1,99 @@
+# ruff: noqa: N802 UP006 UP007
+# @omlish-lite
+import logging
+
+
+class ProxyLogFilterer(logging.Filterer):
+    def __init__(self, underlying: logging.Filterer) -> None:  # noqa
+        self._underlying = underlying
+
+    @property
+    def underlying(self) -> logging.Filterer:
+        return self._underlying
+
+    @property
+    def filters(self):
+        return self._underlying.filters
+
+    @filters.setter
+    def filters(self, filters):
+        self._underlying.filters = filters
+
+    def addFilter(self, filter):  # noqa
+        self._underlying.addFilter(filter)
+
+    def removeFilter(self, filter):  # noqa
+        self._underlying.removeFilter(filter)
+
+    def filter(self, record):
+        return self._underlying.filter(record)
+
+
+class ProxyLogHandler(ProxyLogFilterer, logging.Handler):
+    def __init__(self, underlying: logging.Handler) -> None:  # noqa
+        ProxyLogFilterer.__init__(self, underlying)
+
+    _underlying: logging.Handler
+
+    @property
+    def underlying(self) -> logging.Handler:
+        return self._underlying
+
+    def get_name(self):
+        return self._underlying.get_name()
+
+    def set_name(self, name):
+        self._underlying.set_name(name)
+
+    @property
+    def name(self):
+        return self._underlying.name
+
+    @property
+    def level(self):
+        return self._underlying.level
+
+    @level.setter
+    def level(self, level):
+        self._underlying.level = level
+
+    @property
+    def formatter(self):
+        return self._underlying.formatter
+
+    @formatter.setter
+    def formatter(self, formatter):
+        self._underlying.formatter = formatter
+
+    def createLock(self):
+        self._underlying.createLock()
+
+    def acquire(self):
+        self._underlying.acquire()
+
+    def release(self):
+        self._underlying.release()
+
+    def setLevel(self, level):
+        self._underlying.setLevel(level)
+
+    def format(self, record):
+        return self._underlying.format(record)
+
+    def emit(self, record):
+        self._underlying.emit(record)
+
+    def handle(self, record):
+        return self._underlying.handle(record)
+
+    def setFormatter(self, fmt):
+        self._underlying.setFormatter(fmt)
+
+    def flush(self):
+        self._underlying.flush()
+
+    def close(self):
+        self._underlying.close()
+
+    def handleError(self, record):
+        self._underlying.handleError(record)

omlish-0.0.0.dev158/omlish/logs/standard.py

@@ -0,0 +1,126 @@
+# ruff: noqa: N802 UP006 UP007
+# @omlish-lite
+"""
+TODO:
+ - structured
+ - prefixed
+ - debug
+"""
+import contextlib
+import datetime
+import logging
+import typing as ta
+
+from .filters import TidLogFilter
+from .json import JsonLogFormatter
+from .proxy import ProxyLogHandler
+
+
+##
+
+
+STANDARD_LOG_FORMAT_PARTS = [
+    ('asctime', '%(asctime)-15s'),
+    ('process', 'pid=%(process)-6s'),
+    ('thread', 'tid=%(thread)x'),
+    ('levelname', '%(levelname)s'),
+    ('name', '%(name)s'),
+    ('separator', '::'),
+    ('message', '%(message)s'),
+]
+
+
+class StandardLogFormatter(logging.Formatter):
+    @staticmethod
+    def build_log_format(parts: ta.Iterable[ta.Tuple[str, str]]) -> str:
+        return ' '.join(v for k, v in parts)
+
+    converter = datetime.datetime.fromtimestamp  # type: ignore
+
+    def formatTime(self, record, datefmt=None):
+        ct = self.converter(record.created)  # type: ignore
+        if datefmt:
+            return ct.strftime(datefmt)  # noqa
+        else:
+            t = ct.strftime('%Y-%m-%d %H:%M:%S')
+            return '%s.%03d' % (t, record.msecs)  # noqa
+
+
+##
+
+
+class StandardLogHandler(ProxyLogHandler):
+    pass
+
+
+##
+
+
+@contextlib.contextmanager
+def _locking_logging_module_lock() -> ta.Iterator[None]:
+    if hasattr(logging, '_acquireLock'):
+        logging._acquireLock()  # noqa
+        try:
+            yield
+        finally:
+            logging._releaseLock()  # type: ignore  # noqa
+
+    elif hasattr(logging, '_lock'):
+        # https://github.com/python/cpython/commit/74723e11109a320e628898817ab449b3dad9ee96
+        with logging._lock:  # noqa
+            yield
+
+    else:
+        raise Exception("Can't find lock in logging module")
+
+
+def configure_standard_logging(
+        level: ta.Union[int, str] = logging.INFO,
+        *,
+        json: bool = False,
+        target: ta.Optional[logging.Logger] = None,
+        force: bool = False,
+        handler_factory: ta.Optional[ta.Callable[[], logging.Handler]] = None,
+) -> ta.Optional[StandardLogHandler]:
+    with _locking_logging_module_lock():
+        if target is None:
+            target = logging.root
+
+        #
+
+        if not force:
+            if any(isinstance(h, StandardLogHandler) for h in list(target.handlers)):
+                return None
+
+        #
+
+        if handler_factory is not None:
+            handler = handler_factory()
+        else:
+            handler = logging.StreamHandler()
+
+        #
+
+        formatter: logging.Formatter
+        if json:
+            formatter = JsonLogFormatter()
+        else:
+            formatter = StandardLogFormatter(StandardLogFormatter.build_log_format(STANDARD_LOG_FORMAT_PARTS))
+        handler.setFormatter(formatter)
+
+        #
+
+        handler.addFilter(TidLogFilter())
+
+        #
+
+        target.addHandler(handler)
+
+        #
+
+        if level is not None:
+            target.setLevel(level)
+
+        #
+
+        return StandardLogHandler(handler)