omlish 0.0.0.dev1__py3-none-any.whl

omlish/defs.py

@@ -0,0 +1,193 @@
+"""
+A self-awarely unpythonic set of helpers for defining common boilerplate methods (repr, hash_eq, delegates, etc) in
+class definitions. Should be used sparingly for methods not directly used by humans (like repr) - @property's should
+remain @property's for type annotation, tool assistance, debugging, and otherwise, but these are still nice to have in
+certain circumstances (the real-world alternative usually being simply not adding them).
+"""
+import abc
+import functools
+import operator
+import threading
+
+from . import lang
+
+
+_repr = repr
+
+BASICS = {}
+
+
+def _basic(fn):
+    if fn.__name__ in BASICS:
+        raise NameError(fn.__name__)
+    BASICS[fn.__name__] = fn
+    return fn
+
+
+@lang.cls_dct_fn()
+def basic(cls_dct, *attrs, basics=None):
+    if basics is None:
+        basics = BASICS.keys()
+    for k in basics:
+        fn = BASICS[k]
+        fn(*attrs, cls_dct=cls_dct)
+
+
+_REPR_SEEN = threading.local()
+
+
+def _repr_guard(fn):
+    @functools.wraps(fn)
+    def inner(obj, *args, **kwargs):
+        try:
+            ids = _REPR_SEEN.ids
+        except AttributeError:
+            ids = _REPR_SEEN.ids = set()
+            try:
+                ids.add(id(obj))
+                return fn(obj, *args, **kwargs)
+            finally:
+                del _REPR_SEEN.ids
+        else:
+            if id(obj) in ids:
+                return f'<seen:{type(obj).__name__}@{hex(id(obj))[2:]}>'
+            ids.add(id(obj))
+            return fn(obj, *args, **kwargs)
+
+    return inner
+
+
+@_repr_guard
+def build_attr_repr(obj, *, mro=False):
+    if mro:
+        attrs = [
+            attr
+            for ty in sorted(reversed(type(obj).__mro__), key=lambda _ty: _ty.__dict__.get('__repr_priority__', 0))
+            for attr in ty.__dict__.get('__repr_attrs__', [])]
+    else:
+        attrs = obj.__repr_attrs__
+    return '%s@%s(%s)' % (
+        type(obj).__name__,
+        hex(id(obj))[2:],
+        ', '.join('%s=%s' % (attr, '<self>' if value is obj else _repr(value))
+                  for attr in attrs for value in [getattr(obj, attr)]))
+
+
+@_repr_guard
+def build_repr(obj, *attrs):
+    return '%s@%s(%s)' % (
+        type(obj).__name__,
+        hex(id(obj))[2:],
+        ', '.join('%s=%r' % (attr, getattr(obj, attr)) for attr in attrs))
+
+
+@_basic
+@lang.cls_dct_fn()
+def repr(cls_dct, *attrs, mro=False, priority=None):
+    def __repr__(self):
+        return build_attr_repr(self, mro=mro)
+
+    cls_dct['__repr_attrs__'] = attrs
+    if priority is not None:
+        cls_dct['__repr_priority__'] = priority
+    cls_dct['__repr__'] = __repr__
+
+
+@lang.cls_dct_fn()
+def bare_repr(cls_dct, *attrs):
+    def __repr__(self):
+        return lang.attr_repr(self, *attrs)
+
+    cls_dct['__repr__'] = __repr__
+
+
+@lang.cls_dct_fn()
+def name_repr(cls_dct):
+    def __repr__(self):
+        return self.__name__
+
+    cls_dct['__repr__'] = __repr__
+
+
+@lang.cls_dct_fn()
+def ne(cls_dct):
+    def __ne__(self, other):
+        return not (self == other)
+
+    cls_dct['__ne__'] = __ne__
+
+
+@lang.cls_dct_fn()
+def no_order(cls_dct, *, raise_=None):
+    def fn(self, other):
+        if raise_ is None:
+            return NotImplemented
+        else:
+            raise raise_
+
+    for att in [
+        '__lt__',
+        '__le__',
+        '__gt__',
+        '__ge__',
+    ]:
+        cls_dct[att] = fn
+
+
+@_basic
+@lang.cls_dct_fn()
+def hash_eq(cls_dct, *attrs):
+    def __hash__(self):
+        return hash(tuple(getattr(self, attr) for attr in attrs))
+
+    cls_dct['__hash__'] = __hash__
+
+    def __eq__(self, other):
+        if type(other) is not type(self):
+            return False
+        for attr in attrs:
+            if getattr(self, attr) != getattr(other, attr):
+                return False
+        return True
+
+    cls_dct['__eq__'] = __eq__
+
+    ne(cls_dct=cls_dct)
+
+
+@lang.cls_dct_fn()
+def getter(cls_dct, *attrs):
+    for attr in attrs:
+        cls_dct[attr] = property(operator.attrgetter('_' + attr))
+
+
+@lang.cls_dct_fn()
+def not_implemented(cls_dct, *names, **kwargs):
+    wrapper = kwargs.pop('wrapper', lambda _: _)
+    if kwargs:
+        raise TypeError(kwargs)
+    ret = []
+    for name in names:
+        @wrapper
+        def not_implemented(self, *args, **kwargs):
+            raise NotImplementedError
+
+        not_implemented.__name__ = name
+        cls_dct[name] = not_implemented
+        ret.append(not_implemented)
+    return tuple(ret)
+
+
+@lang.cls_dct_fn()
+def abstract_method(cls_dct, *names):
+    return not_implemented(cls_dct, *names, wrapper=abc.abstractmethod)
+
+
+@lang.cls_dct_fn()
+def abstract_property(cls_dct, *names):
+    return not_implemented(cls_dct, *names, wrapper=abc.abstractmethod)
+
+
+@lang.cls_dct_fn()
+def abstract_hash_eq(cls_dct):
+    return not_implemented(cls_dct, '__hash__', '__eq__', '__ne__', wrapper=abc.abstractmethod)

omlish/dispatch/__init__.py

@@ -0,0 +1,3 @@
+from .dispatch import Dispatcher  # noqa
+from .functions import function  # noqa
+from .methods import method  # noqa

omlish/dispatch/dispatch.py

@@ -0,0 +1,137 @@
+import abc
+import typing as ta
+import weakref
+
+from .. import c3
+from .. import check
+from .. import reflect as rfl
+
+
+T = ta.TypeVar('T')
+
+
+##
+
+
+_IMPL_FUNC_CLS_SET_CACHE: ta.MutableMapping[ta.Callable, ta.FrozenSet[type]] = weakref.WeakKeyDictionary()
+
+
+def get_impl_func_cls_set(func: ta.Callable) -> ta.FrozenSet[type]:
+    try:
+        return _IMPL_FUNC_CLS_SET_CACHE[func]
+    except KeyError:
+        pass
+
+    ann = getattr(func, '__annotations__', {})
+    if not ann:
+        raise TypeError(f'Invalid impl func: {func!r}')
+
+    _, cls = next(iter(ta.get_type_hints(func).items()))
+    if rfl.is_union_type(cls):
+        ret = frozenset(check.isinstance(arg, type) for arg in ta.get_args(cls))
+    else:
+        ret = frozenset([check.isinstance(cls, type)])
+
+    _IMPL_FUNC_CLS_SET_CACHE[func] = ret
+    return ret
+
+
+def find_impl(cls: type, registry: ta.Mapping[type, T]) -> ta.Optional[T]:
+    mro = c3.compose_mro(cls, registry.keys())
+
+    match: ta.Optional[type] = None
+    for t in mro:
+        if match is not None:
+            # If *match* is an implicit ABC but there is another unrelated, equally matching implicit ABC, refuse the
+            # temptation to guess.
+            if (
+                    t in registry
+                    and t not in cls.__mro__
+                    and match not in cls.__mro__
+                    and not issubclass(match, t)
+            ):
+                raise RuntimeError(f'Ambiguous dispatch: {match} or {t}')
+            break
+
+        if t in registry:
+            match = t
+
+    if match is None:
+        return None
+    return registry.get(match)
+
+
+##
+
+
+class Dispatcher(ta.Generic[T]):
+    def __init__(self) -> None:
+        super().__init__()
+
+        impls_by_arg_cls: dict[type, T] = {}
+        self._impls_by_arg_cls = impls_by_arg_cls
+
+        dispatch_cache: dict[ta.Any, ta.Optional[T]] = {}
+        self._get_dispatch_cache = lambda: dispatch_cache
+
+        def cache_remove(k, self_ref=weakref.ref(self)):
+            if (ref_self := self_ref()) is not None:
+                cache = ref_self._get_dispatch_cache()  # noqa
+                try:
+                    del cache[k]
+                except KeyError:
+                    pass
+
+        cache_token: ta.Any = None
+        self._get_cache_token = lambda: cache_token
+
+        weakref_ref_ = weakref.ref
+
+        def dispatch(cls: type) -> ta.Optional[T]:
+            nonlocal cache_token
+
+            if cache_token is not None and (current_token := abc.get_cache_token()) != cache_token:
+                dispatch_cache.clear()
+                cache_token = current_token
+
+            cls_ref = weakref_ref_(cls)
+            try:
+                return dispatch_cache[cls_ref]
+            except KeyError:
+                pass
+            del cls_ref
+
+            impl: T | None
+            try:
+                impl = impls_by_arg_cls[cls]
+            except KeyError:
+                impl = find_impl(cls, impls_by_arg_cls)
+
+            dispatch_cache[weakref_ref_(cls, cache_remove)] = impl
+            return impl
+
+        self.dispatch = dispatch
+
+        def register(impl: T, cls_col: ta.Iterable[type]) -> T:
+            nonlocal cache_token
+
+            for cls in cls_col:
+                impls_by_arg_cls[cls] = impl
+
+                if cache_token is None and hasattr(cls, '__abstractmethods__'):
+                    cache_token = abc.get_cache_token()
+
+            dispatch_cache.clear()
+            return impl
+
+        self.register = register
+
+    _get_cache_token: ta.Callable[[], int]
+    _get_dispatch_cache: ta.Callable[[], ta.Any]
+
+    def cache_size(self) -> int:
+        return len(self._get_dispatch_cache())
+
+    dispatch: ta.Callable[[type], ta.Optional[T]]
+
+    register: ta.Callable[[T, ta.Iterable[type]], T]

omlish/dispatch/functions.py

@@ -0,0 +1,52 @@
+import functools
+import typing as ta
+
+from .. import check
+from .dispatch import Dispatcher
+from .dispatch import get_impl_func_cls_set
+
+
+# USE_EXTENSION = True
+USE_EXTENSION = False
+
+
+def function(func):
+    disp = Dispatcher()  # type: ignore
+    disp.register(func, [object])
+
+    func_name = getattr(func, '__name__', 'singledispatch function')
+
+    def register(impl, cls=None):
+        check.callable(impl)
+        cls_col: ta.Iterable[type]
+        if cls is None:
+            cls_col = get_impl_func_cls_set(impl)
+        else:
+            cls_col = frozenset([cls])
+        disp.register(impl, cls_col)
+        return impl
+
+    if not USE_EXTENSION:
+        disp_dispatch = disp.dispatch
+
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            if not args:
+                raise TypeError(f'{func_name} requires at least 1 positional argument')
+            if (impl := disp_dispatch(type(args[0]))) is not None:
+                return impl(*args, **kwargs)
+            raise RuntimeError(f'No dispatch: {type(args[0])}')
+        wrapper.register = register  # type: ignore
+        wrapper.dispatch = disp.dispatch  # type: ignore
+
+    else:
+        from x.c._dispatch import function_wrapper  # noqa
+        wrapper = function_wrapper(
+            disp.dispatch,
+            **{k: getattr(func, k) for k in functools.WRAPPER_ASSIGNMENTS if hasattr(func, k)},
+            __wrapped__=func,
+            register=register,
+            dispatcher=disp,
+        )
+
+    return wrapper

omlish/dispatch/methods.py

@@ -0,0 +1,162 @@
+"""
+TODO:
+ - .super(instance[_cls], owner)
+ - ALT: A.f(super(), ... ? :/
+ - classmethod/staticmethod
+"""
+import functools
+import typing as ta
+import weakref
+
+from .. import check
+from .dispatch import Dispatcher
+from .dispatch import get_impl_func_cls_set
+
+
+T = ta.TypeVar('T')
+
+
+def build_mro_dct(instance_cls: type, owner_cls: ta.Optional[type] = None) -> ta.Mapping[str, ta.Any]:
+    if owner_cls is None:
+        owner_cls = instance_cls
+    mro = instance_cls.__mro__[-2::-1]
+    try:
+        pos = mro.index(owner_cls)
+    except ValueError:
+        raise TypeError(f'Owner class {owner_cls} not in mro of instance class {instance_cls}')
+    dct: dict[str, ta.Any] = {}
+    for cur_cls in mro[:pos + 1]:
+        dct.update(cur_cls.__dict__)
+    return dct
+
+
+class Method:
+    def __init__(self, func: ta.Callable) -> None:
+        if not callable(func) and not hasattr(func, '__get__'):  # type: ignore
+            raise TypeError(f'{func!r} is not callable or a descriptor')
+
+        self._func = func
+
+        self._impls: ta.MutableSet[ta.Callable] = weakref.WeakSet()
+
+        # bpo-45678: special-casing for classmethod/staticmethod in Python <=3.9, as functools.update_wrapper doesn't
+        # work properly in singledispatchmethod.__get__ if it is applied to an unbound classmethod/staticmethod
+        unwrapped_func: ta.Any
+        if isinstance(func, (staticmethod, classmethod)):
+            unwrapped_func = func.__func__
+        else:
+            unwrapped_func = func
+        self._unwrapped_func = unwrapped_func
+        self._is_abstractmethod = getattr(func, '__isabstractmethod__', False)  # noqa
+        self.update_wrapper(self)
+
+        self._dispatch_func_cache: dict[ta.Any, ta.Callable] = {}
+
+        def dispatch_func_cache_remove(k, self_ref=weakref.ref(self)):
+            if (ref_self := self_ref()) is not None:
+                cache = ref_self._dispatch_func_cache  # noqa
+                try:
+                    del cache[k]
+                except KeyError:
+                    pass
+
+        self._dispatch_func_cache_remove = dispatch_func_cache_remove
+
+    def __repr__(self) -> str:
+        return f'<{type(self).__module__}.{type(self).__qualname__}:{self._func} at 0x{id(self):x}>'
+
+    def update_wrapper(self, wrapper: T) -> T:
+        for attr in functools.WRAPPER_ASSIGNMENTS:
+            try:
+                value = getattr(self._unwrapped_func, attr)
+            except AttributeError:
+                continue
+            setattr(wrapper, attr, value)
+        setattr(wrapper, '__isabstractmethod__', self._is_abstractmethod)  # noqa
+        return wrapper
+
+    def register(self, impl: T) -> T:
+        # bpo-39679: in Python <= 3.9, classmethods and staticmethods don't inherit __annotations__ of the wrapped
+        # function (fixed in 3.10+ as a side-effect of bpo-43682) but we need that for annotation-derived
+        # singledispatches. So we add that just-in-time here.
+        if isinstance(impl, (staticmethod, classmethod)):
+            impl.__annotations__ = getattr(impl.__func__, '__annotations__', {})
+
+        check.callable(impl)
+        if impl not in self._impls:
+            self._impls.add(impl)  # type: ignore
+            self._dispatch_func_cache.clear()
+
+        return impl
+
+    def build_attr_dispatcher(self, instance_cls: type, owner_cls: ta.Optional[type] = None) -> Dispatcher[str]:
+        disp: Dispatcher[str] = Dispatcher()
+
+        mro_dct = build_mro_dct(instance_cls, owner_cls)
+        seen: ta.Mapping[ta.Any, str] = {}
+        for nam, att in mro_dct.items():
+            if att in self._impls:
+                try:
+                    ex_nam = seen[att]
+                except KeyError:
+                    pass
+                else:
+                    raise TypeError(f'Duplicate impl: {owner_cls} {instance_cls} {nam} {ex_nam}')
+                disp.register(nam, get_impl_func_cls_set(att))
+
+        return disp
+
+    def build_dispatch_func(self, disp: Dispatcher[str]) -> ta.Callable:
+        dispatch = disp.dispatch
+        type_ = type
+        getattr_ = getattr
+        base_func = self._func
+        func_name = getattr(base_func, '__name__', 'singledispatch method')
+
+        def __call__(self, *args, **kwargs):  # noqa
+            if not args:
+                raise TypeError(f'{func_name} requires at least 1 positional argument')
+            if (impl_att := dispatch(type_(args[0]))) is not None:
+                fn = getattr_(self, impl_att)
+                return fn(*args, **kwargs)
+            return base_func.__get__(self)(*args, **kwargs)  # noqa
+
+        self.update_wrapper(__call__)
+        return __call__
+
+    def get_dispatch_func(self, instance_cls: type) -> ta.Callable:
+        cls_ref = weakref.ref(instance_cls)
+        try:
+            return self._dispatch_func_cache[cls_ref]
+        except KeyError:
+            pass
+        del cls_ref
+
+        att_disp = self.build_attr_dispatcher(instance_cls)
+        func = self.build_dispatch_func(att_disp)
+        self._dispatch_func_cache[weakref.ref(instance_cls, self._dispatch_func_cache_remove)] = func
+        return func
+
+    def __get__(self, instance, owner=None):
+        if instance is None:
+            # FIXME: classmethod/staticmethod
+            return self
+
+        instance_cls = type(instance)
+        try:
+            func = self._dispatch_func_cache[weakref.ref(instance_cls)]
+        except KeyError:
+            func = self.get_dispatch_func(instance_cls)
+        return func.__get__(instance, owner)  # noqa
+
+    def __call__(self, instance, *args, **kwargs):
+        instance_cls = type(instance)
+        try:
+            func = self._dispatch_func_cache[weakref.ref(instance_cls)]
+        except KeyError:
+            func = self.get_dispatch_func(instance_cls)
+        return func.__get__(instance)(*args, **kwargs)  # noqa
+
+
+def method(func):
+    return Method(func)

omlish/docker.py

@@ -0,0 +1,149 @@
+"""
+TODO:
+ - merged compose configs: https://github.com/wrmsr/bane/blob/27647abdcfb323b73e6982a5c318c7029496b203/core/dev/docker/compose.go#L38
+ - https://github.com/mag37/dockcheck/blob/3d122f2b868eb53a25a3014f0f6bd499390a3a29/dockcheck.sh
+  - https://github.com/regclient/regclient
+  - https://stackoverflow.com/questions/71409458/how-to-download-docker-image-using-http-api-using-docker-hub-credentials
+  - https://stackoverflow.com/questions/55386202/how-can-i-use-the-docker-registry-api-to-pull-information-about-a-container-get
+  - https://ops.tips/blog/inspecting-docker-image-without-pull/
+  
+repo=library/nginx
+token=$(curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:${repo}:pull" | jq -r '.token')
+curl -H "Authorization: Bearer $token" -s "https://registry-1.docker.io/v2/${repo}/tags/list" | jq
+api="application/vnd.docker.distribution.manifest.v2+json"
+apil="application/vnd.docker.distribution.manifest.list.v2+json"
+curl -H "Accept: ${api}" -H "Accept: ${apil}" -H "Authorization: Bearer $token" -s "https://registry-1.docker.io/v2/${repo}/manifests/latest" | jq .
+"""  # noqa
+import datetime
+import re
+import shlex
+import subprocess
+import typing as ta
+
+from . import check
+from . import dataclasses as dc
+from . import json
+from . import lang
+from . import marshal as msh
+
+if ta.TYPE_CHECKING:
+    import yaml
+else:
+    yaml = lang.proxy_import('yaml')
+
+
+@dc.dataclass(frozen=True)
+class PsItem(lang.Final):
+    dc.metadata(msh.ObjectMetadata(
+        field_naming=msh.Naming.CAMEL,
+        unknown_field='x',
+    ))
+
+    command: str
+    created_at: datetime.datetime
+    id: str = dc.field(metadata={msh.FieldMetadata: msh.FieldMetadata(name='ID')})
+    image: str
+    labels: str
+    local_volumes: str
+    mounts: str
+    names: str
+    networks: str
+    ports: str
+    running_for: str
+    size: str
+    state: str
+    status: str
+
+    x: ta.Mapping[str, ta.Any] | None = None
+
+
+class Port(ta.NamedTuple):
+    ip: str
+    from_port: int
+    to_port: int
+    proto: str
+
+
+_PORT_PAT = re.compile(r'(?P<ip>[^:]+):(?P<from_port>\d+)->(?P<to_port>\d+)/(?P<proto>\w+)')
+
+
+def parse_port(s: str) -> Port:
+    # '0.0.0.0:35221->22/tcp, 0.0.0.0:35220->8000/tcp'
+    m = check.not_none(_PORT_PAT.fullmatch(s))
+    return Port(
+        m.group('ip'),
+        int(m.group('from_port')),
+        int(m.group('to_port')),
+        m.group('proto'),
+    )
+
+
+def cli_ps() -> list[PsItem]:
+    o = subprocess.check_output([
+        'docker',
+        'ps',
+        '--no-trunc',
+        '--format', '{{json .}}',
+    ])
+
+    ret: list[PsItem] = []
+    for l in o.decode().splitlines():
+        d = json.loads(l)
+        pi = msh.unmarshal(d, PsItem)
+        ret.append(pi)
+
+    return ret
+
+
+@dc.dataclass(frozen=True)
+class Inspect(lang.Final):
+    dc.metadata(msh.ObjectMetadata(
+        field_naming=msh.Naming.CAMEL,
+        unknown_field='x',
+    ))
+
+    id: str
+    created: datetime.datetime
+
+    x: ta.Mapping[str, ta.Any] | None = None
+
+
+def cli_inspect(ids: list[str]) -> list[Inspect]:
+    o = subprocess.check_output(['docker', 'inspect', *ids])
+    return msh.unmarshal(json.loads(o.decode()), list[Inspect])
+
+
+class ComposeConfig:
+    def __init__(
+            self,
+            prefix: str,
+            *,
+            compose_path: str | None = None,
+    ) -> None:
+        super().__init__()
+
+        self._prefix = prefix
+        self._compose_path = compose_path
+
+    @lang.cached_function
+    def get_config(self) -> ta.Mapping[str, ta.Any]:
+        with open(check.not_none(self._compose_path), 'r') as f:
+            buf = f.read()
+        dct = yaml.safe_load(buf)
+
+        ret = {}
+        for n, c in dct['services'].items():
+            check.state(n.startswith(self._prefix))
+            ret[n[len(self._prefix):]] = c
+
+        return ret
+
+
+def timebomb_payload(delay_s: int | float, name: str = 'omlish-timebomb') -> str:
+    return (
+        '('
+        f'echo {shlex.quote(name)} && '
+        f'sleep {delay_s:g} && '
+        'sh -c \'killall5 -9 -o $PPID -o $$ ; kill 1\''
+        ') &'
+    )