omlish 0.0.0.dev18__py3-none-any.whl → 0.0.0.dev20__py3-none-any.whl

omlish/bootstrap/sys.py

@@ -0,0 +1,361 @@
+# ruff: noqa: UP006 UP007
+import contextlib
+import dataclasses as dc
+import enum
+import faulthandler
+import gc
+import importlib
+import logging
+import os
+import pwd
+import resource
+import signal
+import sys
+import typing as ta
+
+from .. import lang
+from .base import Bootstrap
+from .base import ContextBootstrap
+from .base import SimpleBootstrap
+
+
+if ta.TYPE_CHECKING:
+    from .. import libc
+    from .. import logs
+    from .. import os as osu
+    from ..formats import dotenv
+
+else:
+    libc = lang.proxy_import('..libc', __package__)
+    logs = lang.proxy_import('..logs', __package__)
+    osu = lang.proxy_import('..os', __package__)
+    dotenv = lang.proxy_import('.formats.dotenv', __package__)
+
+
+##
+
+
+class CwdBootstrap(ContextBootstrap['CwdBootstrap.Config']):
+    @dc.dataclass(frozen=True)
+    class Config(Bootstrap.Config):
+        path: ta.Optional[str] = None
+
+    @contextlib.contextmanager
+    def enter(self) -> ta.Iterator[None]:
+        if self._config.path is not None:
+            prev = os.getcwd()
+            os.chdir(self._config.path)
+        else:
+            prev = None
+
+        try:
+            yield
+
+        finally:
+            if prev is not None:
+                os.chdir(prev)
+
+
+##
+
+
+class SetuidBootstrap(SimpleBootstrap['SetuidBootstrap.Config']):
+    @dc.dataclass(frozen=True)
+    class Config(Bootstrap.Config):
+        user: ta.Optional[str] = None
+
+    def run(self) -> None:
+        if self._config.user is not None:
+            user = pwd.getpwnam(self._config.user)
+            os.setuid(user.pw_uid)
+
+
+##
+
+
+class GcDebugFlag(enum.Enum):
+    STATS = gc.DEBUG_STATS
+    COLLECTABLE = gc.DEBUG_COLLECTABLE
+    UNCOLLECTABLE = gc.DEBUG_UNCOLLECTABLE
+    SAVEALL = gc.DEBUG_SAVEALL
+    LEAK = gc.DEBUG_LEAK
+
+
+class GcBootstrap(ContextBootstrap['GcBootstrap.Config']):
+    @dc.dataclass(frozen=True)
+    class Config(Bootstrap.Config):
+        disable: bool = False
+        debug: ta.Optional[int] = None
+
+    @contextlib.contextmanager
+    def enter(self) -> ta.Iterator[None]:
+        prev_enabled = gc.isenabled()
+        if self._config.disable:
+            gc.disable()
+
+        if self._config.debug is not None:
+            prev_debug = gc.get_debug()
+            gc.set_debug(self._config.debug)
+        else:
+            prev_debug = None
+
+        try:
+            yield
+
+        finally:
+            if prev_enabled:
+                gc.enable()
+
+            if prev_debug is not None:
+                gc.set_debug(prev_debug)
+
+
+##
+
+
+class NiceBootstrap(SimpleBootstrap['NiceBootstrap.Config']):
+    @dc.dataclass(frozen=True)
+    class Config(Bootstrap.Config):
+        nice: ta.Optional[int] = None
+
+    def run(self) -> None:
+        if self._config.nice is not None:
+            os.nice(self._config.nice)
+
+
+##
+
+
+class LogBootstrap(ContextBootstrap['LogBootstrap.Config']):
+    @dc.dataclass(frozen=True)
+    class Config(Bootstrap.Config):
+        level: ta.Union[str, int, None] = None
+        json: bool = False
+
+    @contextlib.contextmanager
+    def enter(self) -> ta.Iterator[None]:
+        if self._config.level is None:
+            yield
+            return
+
+        handler = logs.configure_standard_logging(
+            self._config.level,
+            json=self._config.json,
+        )
+
+        try:
+            yield
+
+        finally:
+            if handler is not None:
+                logging.root.removeHandler(handler)
+
+
+##
+
+
+class FaulthandlerBootstrap(ContextBootstrap['FaulthandlerBootstrap.Config']):
+    @dc.dataclass(frozen=True)
+    class Config(Bootstrap.Config):
+        enabled: ta.Optional[bool] = None
+
+    @contextlib.contextmanager
+    def enter(self) -> ta.Iterator[None]:
+        if self._config.enabled is None:
+            yield
+            return
+
+        prev = faulthandler.is_enabled()
+        if self._config.enabled:
+            faulthandler.enable()
+        else:
+            faulthandler.disable()
+
+        try:
+            yield
+
+        finally:
+            if prev:
+                faulthandler.enable()
+            else:
+                faulthandler.disable()
+
+
+##
+
+
+SIGNALS_BY_NAME = {
+    a[len('SIG'):]: v  # noqa
+    for a in dir(signal)
+    if a.startswith('SIG')
+    and not a.startswith('SIG_')
+    and a == a.upper()
+    and isinstance((v := getattr(signal, a)), int)
+}
+
+
+class PrctlBootstrap(SimpleBootstrap['PrctlBootstrap.Config']):
+    @dc.dataclass(frozen=True)
+    class Config(Bootstrap.Config):
+        dumpable: bool = False
+        deathsig: ta.Union[int, str, None] = None
+
+    def run(self) -> None:
+        if self._config.dumpable:
+            libc.prctl(libc.PR_SET_DUMPABLE, 1, 0, 0, 0, 0)
+
+        if self._config.deathsig is not None:
+            if isinstance(self._config.deathsig, int):
+                sig = self._config.deathsig
+            else:
+                sig = SIGNALS_BY_NAME[self._config.deathsig.upper()]
+            libc.prctl(libc.PR_SET_PDEATHSIG, sig, 0, 0, 0, 0)
+
+
+##
+
+
+RLIMITS_BY_NAME = {
+    a[len('RLIMIT_'):]: v  # noqa
+    for a in dir(resource)
+    if a.startswith('RLIMIT_')
+    and a == a.upper()
+    and isinstance((v := getattr(resource, a)), int)
+}
+
+
+class RlimitBootstrap(ContextBootstrap['RlimitBootstrap.Config']):
+    @dc.dataclass(frozen=True)
+    class Config(Bootstrap.Config):
+        limits: ta.Optional[ta.Mapping[str, ta.Tuple[ta.Optional[int], ta.Optional[int]]]] = None
+
+    @contextlib.contextmanager
+    def enter(self) -> ta.Iterator[None]:
+        if not self._config.limits:
+            yield
+            return
+
+        def or_infin(l: ta.Optional[int]) -> int:
+            return l if l is not None else resource.RLIM_INFINITY
+
+        prev = {}
+        for k, (s, h) in self._config.limits.items():
+            i = RLIMITS_BY_NAME[k.upper()]
+            prev[i] = resource.getrlimit(i)
+            resource.setrlimit(i, (or_infin(s), or_infin(h)))
+
+        try:
+            yield
+
+        finally:
+            for i, (s, h) in prev.items():
+                resource.setrlimit(i, (s, h))
+
+
+##
+
+
+class ImportBootstrap(SimpleBootstrap['ImportBootstrap.Config']):
+    @dc.dataclass(frozen=True)
+    class Config(Bootstrap.Config):
+        modules: ta.Optional[ta.Sequence[str]] = None
+
+    def run(self) -> None:
+        for m in self._config.modules or ():
+            importlib.import_module(m)
+
+
+##
+
+
+class EnvBootstrap(ContextBootstrap['EnvBootstrap.Config']):
+    @dc.dataclass(frozen=True)
+    class Config(Bootstrap.Config):
+        vars: ta.Optional[ta.Mapping[str, ta.Optional[str]]] = None
+        files: ta.Optional[ta.Sequence[str]] = None
+
+    @contextlib.contextmanager
+    def enter(self) -> ta.Iterator[None]:
+        if not (self._config.vars or self._config.files):
+            yield
+            return
+
+        new = dict(self._config.vars or {})
+        for f in self._config.files or ():
+            new.update(dotenv.dotenv_values(f, env=os.environ))
+
+        prev: ta.Dict[str, ta.Optional[str]] = {k: os.environ.get(k) for k in new}
+
+        def do(k: str, v: ta.Optional[str]) -> None:
+            if v is not None:
+                os.environ[k] = v
+            else:
+                del os.environ[k]
+
+        for k, v in new.items():
+            do(k, v)
+
+        try:
+            yield
+
+        finally:
+            for k, v in prev.items():
+                do(k, v)
+
+
+##
+
+
+class PidfileBootstrap(ContextBootstrap['PidfileBootstrap.Config']):
+    @dc.dataclass(frozen=True)
+    class Config(Bootstrap.Config):
+        path: ta.Optional[str] = None
+
+    @contextlib.contextmanager
+    def enter(self) -> ta.Iterator[None]:
+        if self._config.path is None:
+            yield
+            return
+
+        with osu.Pidfile(self._config.path) as pf:
+            pf.write()
+            yield
+
+
+##
+
+
+class FdsBootstrap(SimpleBootstrap['FdsBootstrap.Config']):
+    @dc.dataclass(frozen=True)
+    class Config(Bootstrap.Config):
+        redirects: ta.Optional[ta.Mapping[int, ta.Union[int, str, None]]] = None
+
+    def run(self) -> None:
+        for dst, src in (self._config.redirects or {}).items():
+            if src is None:
+                src = '/dev/null'
+            if isinstance(src, int):
+                os.dup2(src, dst)
+            elif isinstance(src, str):
+                sfd = os.open(src, os.O_RDWR)
+                os.dup2(sfd, dst)
+                os.close(sfd)
+            else:
+                raise TypeError(src)
+
+
+##
+
+
+class PrintPidBootstrap(SimpleBootstrap['PrintPidBootstrap.Config']):
+    @dc.dataclass(frozen=True)
+    class Config(Bootstrap.Config):
+        enable: bool = False
+        pause: bool = False
+
+    def run(self) -> None:
+        if not (self._config.enable or self._config.pause):
+            return
+        print(str(os.getpid()), file=sys.stderr)
+        if self._config.pause:
+            input()

omlish/dataclasses/__init__.py

@@ -96,4 +96,6 @@ from .utils import (  # noqa
     opt_repr,
     update_field_extras,
     update_field_metadata,
+    update_fields,
+    update_fields_metadata,
 )

omlish/dataclasses/impl/metadata.py

@@ -70,5 +70,6 @@ class Init(lang.Marker):
     pass
 
 
-def init(fn: ta.Callable[..., None]) -> None:
+def init(fn: ta.Callable):
     _append_cls_md(Init, fn)
+    return fn

omlish/dataclasses/utils.py

@@ -12,6 +12,9 @@ from .impl.params import get_field_extras
 T = ta.TypeVar('T')
 
 
+#
+
+
 def maybe_post_init(sup: ta.Any) -> bool:
     try:
         fn = sup.__post_init__
@@ -21,10 +24,16 @@ def maybe_post_init(sup: ta.Any) -> bool:
     return True
 
 
+#
+
+
 def opt_repr(o: ta.Any) -> str | None:
     return repr(o) if o is not None else None
 
 
+#
+
+
 class field_modifier:  # noqa
     def __init__(self, fn: ta.Callable[[dc.Field], dc.Field]) -> None:
         super().__init__()
@@ -58,6 +67,42 @@ def update_field_extras(f: dc.Field, *, unless_non_default: bool = False, **kwar
     })
 
 
+def update_fields(
+        fn: ta.Callable[[str, dc.Field], dc.Field],
+        fields: ta.Iterable[str] | None = None,
+) -> ta.Callable[[type[T]], type[T]]:
+    def inner(cls):
+        if fields is None:
+            for a, v in list(cls.__dict__.items()):
+                if isinstance(v, dc.Field):
+                    setattr(cls, a, fn(a, v))
+
+        else:
+            for a in fields:
+                v = cls.__dict__[a]
+                if not isinstance(v, dc.Field):
+                    v = dc.field(default=v)
+                setattr(cls, a, fn(a, v))
+
+        return cls
+
+    check.not_isinstance(fields, str)
+    return inner
+
+
+def update_fields_metadata(
+        nmd: ta.Mapping,
+        fields: ta.Iterable[str] | None = None,
+) -> ta.Callable[[type[T]], type[T]]:
+    def inner(a: str, f: dc.Field) -> dc.Field:
+        return update_field_metadata(f, nmd)
+
+    return update_fields(inner, fields)
+
+
+#
+
+
 def deep_replace(o: T, *args: str | ta.Callable[[ta.Any], ta.Mapping[str, ta.Any]]) -> T:
     if not args:
         return o

omlish/defs.py

@@ -5,7 +5,6 @@ remain @property's for type annotation, tool assistance, debugging, and otherwis
 certain circumstances (the real-world alternative usually being simply not adding them).
 """
 # ruff: noqa: ANN201
-
 import abc
 import functools
 import operator
@@ -22,6 +21,7 @@ BASICS = {}
 def _basic(fn):
     if fn.__name__ in BASICS:
         raise NameError(fn.__name__)
+
     BASICS[fn.__name__] = fn
     return fn
 
@@ -30,6 +30,7 @@ def _basic(fn):
 def basic(cls_dct, *attrs, basics=None):
     if basics is None:
         basics = BASICS.keys()
+
     for k in basics:
         fn = BASICS[k]
         fn(*attrs, cls_dct=cls_dct)
@@ -43,6 +44,7 @@ def _repr_guard(fn):
     def inner(obj, *args, **kwargs):
         try:
             ids = _REPR_SEEN.ids
+
         except AttributeError:
             ids = _REPR_SEEN.ids = set()
             try:
@@ -50,6 +52,7 @@ def _repr_guard(fn):
                 return fn(obj, *args, **kwargs)
             finally:
                 del _REPR_SEEN.ids
+
         else:
             if id(obj) in ids:
                 return f'<seen:{type(obj).__name__}@{hex(id(obj))[2:]}>'
@@ -64,17 +67,27 @@ def build_attr_repr(obj, *, mro=False):
     if mro:
         attrs = [
             attr
-            for ty in sorted(reversed(type(obj).__mro__), key=lambda _ty: _ty.__dict__.get('__repr_priority__', 0))  # noqa
-            for attr in ty.__dict__.get('__repr_attrs__', [])]
+            for ty in sorted(  # noqa
+                reversed(type(obj).__mro__),
+                key=lambda _ty: _ty.__dict__.get('__repr_priority__', 0),
+            )
+            for attr in ty.__dict__.get('__repr_attrs__', [])
+        ]
+
     else:
         attrs = obj.__repr_attrs__
+
     s = ', '.join(f'{a}={"<self>" if v is obj else _repr(v)}' for a in attrs for v in [getattr(obj, a)])
     return f'{type(obj).__name__}@{hex(id(obj))[2:]}({s})'
 
 
 @_repr_guard
 def build_repr(obj, *attrs):
-    return f'{type(obj).__name__}@{hex(id(obj))[2:]}({", ".join(f"{attr}={getattr(obj, attr)!r}" for attr in attrs)})'
+    return (
+        f'{type(obj).__name__}'
+        f'@{hex(id(obj))[2:]}'
+        f'({", ".join(f"{attr}={getattr(obj, attr)!r}" for attr in attrs)})'
+    )
 
 
 @_basic
@@ -86,6 +99,7 @@ def repr(cls_dct, *attrs, mro=False, priority=None):  # noqa
     cls_dct['__repr_attrs__'] = attrs
     if priority is not None:
         cls_dct['__repr_priority__'] = priority
+
     cls_dct['__repr__'] = __repr__
 
 
@@ -141,9 +155,11 @@ def hash_eq(cls_dct, *attrs):
     def __eq__(self, other):  # noqa
         if type(other) is not type(self):
             return False
+
         for attr in attrs:  # noqa
             if getattr(self, attr) != getattr(other, attr):
                 return False
+
         return True
 
     cls_dct['__eq__'] = __eq__
@@ -162,6 +178,7 @@ def not_implemented(cls_dct, *names, **kwargs):
     wrapper = kwargs.pop('wrapper', lambda _: _)
     if kwargs:
         raise TypeError(kwargs)
+
     ret = []
     for name in names:
         @wrapper
@@ -171,6 +188,7 @@ def not_implemented(cls_dct, *names, **kwargs):
         not_implemented.__name__ = name
         cls_dct[name] = not_implemented
         ret.append(not_implemented)
+
     return tuple(ret)
 
 
@@ -186,4 +204,10 @@ def abstract_property(cls_dct, *names):
 
 @lang.cls_dct_fn()
 def abstract_hash_eq(cls_dct):
-    return not_implemented(cls_dct, '__hash__', '__eq__', '__ne__', wrapper=abc.abstractmethod)
+    return not_implemented(
+        cls_dct,
+        '__hash__',
+        '__eq__',
+        '__ne__',
+        wrapper=abc.abstractmethod,
+    )

omlish/fnpairs.py

@@ -449,7 +449,7 @@ class Toml(ObjectStr_):
 #
 
 
-@_register_extension('cpkl')
+@_register_extension('clpkl')
 @dc.dataclass(frozen=True)
 class Cloudpickle(ObjectBytes_):
     protocol: int | None = None

omlish/formats/json.py

@@ -13,6 +13,7 @@ from .. import lang
 if ta.TYPE_CHECKING:
     import orjson as _orjson
     import ujson as _ujson
+
 else:
     _orjson = lang.proxy_import('orjson')
     _ujson = lang.proxy_import('ujson')

omlish/lang/descriptors.py

@@ -125,11 +125,13 @@ class _decorator_descriptor:  # noqa
 
         def __get__(self, instance, owner=None):
             fn = self._fn.__get__(instance, owner)
+
             if self._md or instance is not None:
                 @functools.wraps(fn)
                 def inner(*args, **kwargs):
                     return self._wrapper(fn, *args, **kwargs)
                 return inner
+
             else:
                 @functools.wraps(fn)
                 def outer(this, *args, **kwargs):

omlish/lite/logs.py

@@ -69,8 +69,8 @@ class JsonLogFormatter(logging.Formatter):
 STANDARD_LOG_FORMAT_PARTS = [
     ('asctime', '%(asctime)-15s'),
     ('process', 'pid=%(process)-6s'),
-    ('thread', 'tid=%(thread)-16s'),
-    ('levelname', '%(levelname)-8s'),
+    ('thread', 'tid=%(thread)x'),
+    ('levelname', '%(levelname)s'),
     ('name', '%(name)s'),
     ('separator', '::'),
     ('message', '%(message)s'),

omlish/logs/configs.py

@@ -3,21 +3,16 @@ import logging
 import typing as ta
 
 from ..lite.logs import configure_standard_logging as configure_lite_standard_logging
+from .noisy import silence_noisy_loggers
 
 
 ##
 
 
-NOISY_LOGGERS: set[str] = {
-    'boto3.resources.action',
-    'datadog.dogstatsd',
-    'elasticsearch',
-    'kazoo.client',
-    'requests.packages.urllib3.connectionpool',
-}
-
-
-##
+FilterConfig = dict[str, ta.Any]
+FormatterConfig = dict[str, ta.Any]
+HandlerConfig = dict[str, ta.Any]
+LoggerConfig = dict[str, ta.Any]
 
 
 @dc.dataclass()
@@ -25,17 +20,11 @@ class DictConfig:
     version: int = 1
     incremental: bool = False
     disable_existing_loggers: bool = False
-    filters: dict[str, 'FilterConfig'] = dc.field(default_factory=dict)
-    formatters: dict[str, 'FormatterConfig'] = dc.field(default_factory=dict)
-    handlers: dict[str, 'HandlerConfig'] = dc.field(default_factory=dict)
-    loggers: dict[str, 'LoggerConfig'] = dc.field(default_factory=dict)
-    root: ta.Optional['LoggerConfig'] = None
-
-
-FilterConfig = dict[str, ta.Any]
-FormatterConfig = dict[str, ta.Any]
-HandlerConfig = dict[str, ta.Any]
-LoggerConfig = dict[str, ta.Any]
+    filters: dict[str, FilterConfig] = dc.field(default_factory=dict)
+    formatters: dict[str, FormatterConfig] = dc.field(default_factory=dict)
+    handlers: dict[str, HandlerConfig] = dc.field(default_factory=dict)
+    loggers: dict[str, LoggerConfig] = dc.field(default_factory=dict)
+    root: LoggerConfig | None = None
 
 
 ##
@@ -51,7 +40,6 @@ def configure_standard_logging(
         json=json,
     )
 
-    for noisy_logger in NOISY_LOGGERS:
-        logging.getLogger(noisy_logger).setLevel(logging.WARNING)
+    silence_noisy_loggers()
 
     return handler

omlish/logs/noisy.py

@@ -0,0 +1,15 @@
+import logging
+
+
+NOISY_LOGGERS: set[str] = {
+    'boto3.resources.action',
+    'datadog.dogstatsd',
+    'elasticsearch',
+    'kazoo.client',
+    'requests.packages.urllib3.connectionpool',
+}
+
+
+def silence_noisy_loggers() -> None:
+    for noisy_logger in NOISY_LOGGERS:
+        logging.getLogger(noisy_logger).setLevel(logging.WARNING)

omlish/marshal/__init__.py

@@ -48,6 +48,10 @@ from .global_ import (  # noqa
     unmarshal,
 )
 
+from .helpers import (  # noqa
+    update_fields_metadata,
+)
+
 from .objects import (  # noqa
     FieldMetadata,
     ObjectMetadata,