omlish 0.0.0.dev1__py3-none-any.whl

omlish/__about__.py

@@ -0,0 +1,7 @@
+__name__ = 'omlish'
+__author__ = 'wrmsr'
+__url__ = 'https://github.com/wrmsr/omlish'
+__license__ = 'BSD-3-Clause'
+__requires_python__ = '>=3.11'
+
+__version__ = '0.0.0.dev1'

omlish/argparse.py

@@ -0,0 +1,223 @@
+"""
+TODO:
+ - default command
+"""
+import argparse
+import dataclasses as dc
+import functools
+import sys
+import typing as ta
+
+from . import c3
+from . import check
+
+
+T = ta.TypeVar('T')
+
+
+SUPPRESS = argparse.SUPPRESS
+
+OPTIONAL = argparse.OPTIONAL
+ZERO_OR_MORE = argparse.ZERO_OR_MORE
+ONE_OR_MORE = argparse.ONE_OR_MORE
+PARSER = argparse.PARSER
+REMAINDER = argparse.REMAINDER
+
+HelpFormatter = argparse.HelpFormatter
+RawDescriptionHelpFormatter = argparse.RawDescriptionHelpFormatter
+RawTextHelpFormatter = argparse.RawTextHelpFormatter
+ArgumentDefaultsHelpFormatter = argparse.ArgumentDefaultsHelpFormatter
+
+MetavarTypeHelpFormatter = argparse.MetavarTypeHelpFormatter
+
+ArgumentError = argparse.ArgumentError
+ArgumentTypeError = argparse.ArgumentTypeError
+
+Action = argparse.Action
+BooleanOptionalAction = argparse.BooleanOptionalAction
+SubParsersAction = argparse._SubParsersAction  # noqa
+
+FileType = argparse.FileType
+
+Namespace = argparse.Namespace
+
+ArgumentParser = argparse.ArgumentParser
+
+
+@dc.dataclass(eq=False)
+class Arg:
+    args: ta.Sequence[ta.Any]
+    kwargs: ta.Mapping[str, ta.Any]
+    dest: ta.Optional[str] = None
+
+    def __get__(self, instance, owner=None):
+        if instance is None:
+            return self
+        return getattr(instance.args, self.dest)  # type: ignore
+
+
+def arg(*args, **kwargs) -> Arg:
+    return Arg(args, kwargs)
+
+
+CommandFn = ta.Callable[[], None]
+
+
+@dc.dataclass(eq=False)
+class Command:
+    name: str
+    fn: CommandFn
+    args: ta.Sequence[Arg] = ()
+    parent: ta.Optional['Command'] = None
+
+    def __post_init__(self) -> None:
+        check.isinstance(self.name, str)
+        check.not_in('-', self.name)
+        check.not_empty(self.name)
+
+        check.callable(self.fn)
+        check.arg(all(isinstance(a, Arg) for a in self.args))
+        check.isinstance(self.parent, (Command, None))
+
+        functools.update_wrapper(self, self.fn)
+
+    def __get__(self, instance, owner=None):
+        if instance is None:
+            return self
+        return dc.replace(self, fn=self.fn.__get__(instance, owner))
+
+    def __call__(self, *args, **kwargs) -> None:
+        return self.fn(*args, **kwargs)
+
+
+def command(
+        *args: Arg,
+        name: ta.Optional[str] = None,
+        parent: ta.Optional[Command] = None,
+) -> ta.Any:  # ta.Callable[[CommandFn], Command]:  # FIXME
+    for arg in args:
+        check.isinstance(arg, Arg)
+    check.isinstance(name, (str, None))
+    check.isinstance(parent, (Command, None))
+
+    def inner(fn):
+        return Command(
+            (name if name is not None else fn.__name__).replace('-', '_'),
+            fn,
+            args,
+            parent=parent,
+        )
+
+    return inner
+
+
+def get_arg_ann_kwargs(ann: ta.Any) -> ta.Mapping[str, ta.Any]:
+    if ann is str:
+        return {}
+    elif ann is int:
+        return {'type': int}
+    elif ann is bool:
+        return {'action': 'store_true'}
+    elif ann is list:
+        return {'action': 'append'}
+    else:
+        raise TypeError(ann)
+
+
+class _AnnotationBox:
+
+    def __init__(self, annotations: ta.Mapping[str, ta.Any]) -> None:
+        super().__init__()
+        self.__annotations__ = annotations  # type: ignore
+
+
+class _CliMeta(type):
+
+    def __new__(mcls, name: str, bases: ta.Sequence[type], namespace: ta.Mapping[str, ta.Any]) -> type:
+        if not bases:
+            return super().__new__(mcls, name, tuple(bases), dict(namespace))
+
+        bases = list(bases)
+        namespace = dict(namespace)
+
+        objs = {}
+        mro = c3.merge([list(b.__mro__) for b in bases])
+        for bns in [bcls.__dict__ for bcls in reversed(mro)] + [namespace]:
+            bseen = set()  # type: ignore
+            for k, v in bns.items():
+                if isinstance(v, (Command, Arg)):
+                    check.not_in(v, bseen)
+                    bseen.add(v)
+                    objs[k] = v
+                elif k in objs:
+                    del [k]
+
+        anns = ta.get_type_hints(_AnnotationBox({
+            **{k: v for bcls in reversed(mro) for k, v in getattr(bcls, '__annotations__', {}).items()},
+            **namespace.get('__annotations__', {}),
+        }), globalns=namespace.get('__globals__', {}))
+
+        if 'parser' in namespace:
+            parser = check.isinstance(namespace.pop('parser'), ArgumentParser)
+        else:
+            parser = ArgumentParser()
+        namespace['_parser'] = parser
+
+        subparsers = parser.add_subparsers()
+        for name, obj in objs.items():
+            if isinstance(obj, Command):
+                if obj.parent is not None:
+                    raise NotImplementedError
+                cparser = subparsers.add_parser(obj.name)
+                for arg in (obj.args or []):
+                    cparser.add_argument(*arg.args, **arg.kwargs)
+                cparser.set_defaults(_cmd=obj)
+
+            elif isinstance(obj, Arg):
+                if name in anns:
+                    akwargs = get_arg_ann_kwargs(anns[name])
+                    obj.kwargs = {**akwargs, **obj.kwargs}
+                if not obj.dest:
+                    if 'dest' in obj.kwargs:
+                        obj.dest = obj.kwargs['dest']
+                    else:
+                        obj.dest = obj.kwargs['dest'] = name  # type: ignore
+                parser.add_argument(*obj.args, **obj.kwargs)
+
+            else:
+                raise TypeError(obj)
+
+        return super().__new__(mcls, name, tuple(bases), namespace)
+
+
+class Cli(metaclass=_CliMeta):
+
+    def __init__(self, argv: ta.Optional[ta.Sequence[str]] = None) -> None:
+        super().__init__()
+
+        self._argv = argv if argv is not None else sys.argv[1:]
+        self._args = self.get_parser().parse_args(self._argv)
+
+    _parser: ta.ClassVar[ArgumentParser]
+
+    @classmethod
+    def get_parser(cls) -> ArgumentParser:
+        return cls._parser
+
+    @property
+    def argv(self) -> ta.Sequence[str]:
+        return self._argv
+
+    @property
+    def args(self) -> Namespace:
+        return self._args
+
+    def _run_cmd(self, cmd: Command) -> None:
+        cmd.__get__(self, type(self))()
+
+    def __call__(self) -> None:
+        cmd = getattr(self.args, '_cmd', None)
+        if cmd is None:
+            self.get_parser().print_help()
+            return
+        self._run_cmd(cmd)

omlish/asyncs/__init__.py

@@ -0,0 +1,17 @@
+from .asyncs import (  # noqa
+    SyncableIterable,
+    async_list,
+    sync_await,
+    sync_list,
+    syncable_iterable,
+)
+
+
+from .futures import (  # noqa
+    FutureException,
+    FutureTimeoutException,
+    ImmediateExecutor,
+    new_thread_or_immediate_executor,
+    wait_dependent_futures,
+    wait_futures,
+)

omlish/asyncs/anyio.py

@@ -0,0 +1,23 @@
+"""
+lookit:
+ - https://github.com/davidbrochart/sqlite-anyio/blob/a3ba4c6ef0535b14a5a60071fcd6ed565a514963/sqlite_anyio/sqlite.py
+ - https://github.com/rafalkrupinski/ratelimit-anyio/blob/2910a8a3d6fa54ed17ee6ba457686c9f7a4c4beb/src/ratelimit_anyio/__init__.py
+ - https://github.com/nekitdev/async-extensions/tree/main/async_extensions
+ - https://github.com/kinnay/anynet/tree/master/anynet
+ - https://github.com/M-o-a-T/asyncscope
+ - https://github.com/M-o-a-T/aevent
+ - https://github.com/florimondmanca/aiometer
+"""  # noqa
+import typing as ta
+
+import anyio
+
+
+T = ta.TypeVar('T')
+
+
+async def anyio_eof_to_empty(fn: ta.Callable[..., ta.Awaitable[T]], *args: ta.Any, **kwargs: ta.Any) -> T | bytes:
+    try:
+        return await fn(*args, **kwargs)
+    except anyio.EndOfStream:
+        return b''

omlish/asyncs/asyncio.py

@@ -0,0 +1,19 @@
+import asyncio
+import functools
+import typing as ta
+
+
+CallableT = ta.TypeVar('CallableT', bound=ta.Callable)
+
+
+def asyncio_once(fn: CallableT) -> CallableT:
+    future = None
+
+    @functools.wraps(fn)
+    async def inner(*args, **kwargs):
+        nonlocal future
+        if not future:
+            future = asyncio.create_task(fn(*args, **kwargs))
+        return await future
+
+    return ta.cast(CallableT, inner)

omlish/asyncs/asyncs.py

@@ -0,0 +1,76 @@
+"""
+TODO:
+ - async<->sync greeenlet bridge
+  In [5]: %timeit greenlet.greenlet(f).switch()
+  517 ns ± 13.2 ns per loop (mean ± std. dev. of 7 runs, 1000000 loops each)
+  - injected io provider - sync vs greenlet aio trampolined
+ - push/pull bridge?
+
+https://github.com/sqlalchemy/sqlalchemy/blob/1e75c189da721395bc8c2d899c722a5b9a170404/lib/sqlalchemy/util/_concurrency_py3k.py#L83
+"""
+import contextlib
+import functools
+import typing as ta
+
+
+T = ta.TypeVar('T')
+
+
+def sync_await(fn: ta.Callable[..., T], *args, **kwargs) -> T:
+    ret: ta.Any
+    ret = missing = object()
+
+    async def gate():
+        nonlocal ret
+        ret = await fn(*args, **kwargs)  # type: ignore
+
+    cr = gate()
+    with contextlib.closing(cr):
+        try:
+            cr.send(None)
+        except StopIteration:
+            pass
+        if ret is missing or cr.cr_await is not None or cr.cr_running:
+            raise TypeError('Not terminated')
+
+    return ta.cast(T, ret)
+
+
+def sync_list(fn: ta.Callable[..., ta.AsyncIterator[T]], *args, **kwargs) -> list[T]:
+    lst = None
+
+    async def inner():
+        nonlocal lst
+        lst = [v async for v in fn(*args, **kwargs)]
+
+    sync_await(inner)
+    if not isinstance(lst, list):
+        raise TypeError(lst)
+    return lst
+
+
+async def async_list(fn: ta.Callable[..., ta.AsyncIterator[T]], *args, **kwargs) -> list[T]:
+    return [v async for v in fn(*args, **kwargs)]
+
+
+class SyncableIterable(ta.Generic[T]):
+
+    def __init__(self, obj) -> None:
+        super().__init__()
+        self._obj = obj
+
+    def __iter__(self) -> ta.Iterator[T]:
+        async def inner():
+            async for i in self._obj:
+                yield i
+        return iter(sync_list(inner))
+
+    def __aiter__(self) -> ta.AsyncIterator[T]:
+        return self._obj.__aiter__()
+
+
+def syncable_iterable(fn: ta.Callable[..., ta.AsyncIterator[T]]) -> ta.Callable[..., SyncableIterable[T]]:
+    @functools.wraps(fn)
+    def inner(*args, **kwargs):
+        return SyncableIterable(fn(*args, **kwargs))
+    return inner

omlish/asyncs/futures.py

@@ -0,0 +1,179 @@
+import concurrent.futures as cf
+import contextlib
+import time
+import typing as ta
+
+
+T = ta.TypeVar('T')
+
+
+##
+
+
+class FutureException(Exception, ta.Generic[T]):
+
+    def __init__(self, future: cf.Future, target: ta.Optional[T] = None) -> None:
+        super().__init__()
+
+        self._future = future
+        self._target = target
+
+    @property
+    def future(self) -> cf.Future:
+        return self._future
+
+    @property
+    def target(self) -> ta.Optional[T]:
+        return self._target
+
+    def __repr__(self) -> str:
+        return (
+            f'{self.__class__.__qualname__}('
+            f'exception={self._future.exception()!r}, '
+            f'future={self._future!r}, '
+            f'target={self._target})'
+        )
+
+    __str__ = __repr__
+
+
+class FutureTimeoutException(Exception):
+    pass
+
+
+def wait_futures(
+        futures: ta.Sequence[cf.Future],
+        *,
+        timeout_s: ta.Union[int, float] = 60,
+        tick_interval_s: ta.Union[int, float] = 0.5,
+        tick_fn: ta.Callable[..., bool] = lambda: True,
+        raise_exceptions: bool = False,
+        cancel_on_exception: bool = False,
+) -> bool:
+    start = time.time()
+
+    not_done = set(futures)
+    while tick_fn():
+        done = {f for f in not_done if f.done()}
+        if raise_exceptions:
+            for fut in done:
+                if fut.exception():
+                    if cancel_on_exception:
+                        for cancel_fut in not_done:
+                            cancel_fut.cancel()
+                    raise FutureException(fut) from fut.exception()
+
+        not_done -= done
+        if not not_done:
+            return True
+
+        if time.time() >= (start + timeout_s):
+            raise FutureTimeoutException
+        time.sleep(tick_interval_s)
+
+    return False
+
+
+def wait_dependent_futures(
+        executor: cf.Executor,
+        dependency_sets_by_fn: ta.Mapping[ta.Callable, ta.AbstractSet[ta.Callable]],
+        *,
+        timeout_s: ta.Union[int, float] = 60,
+        tick_interval_s: ta.Union[int, float] = 0.5,
+        tick_fn: ta.Callable[..., bool] = lambda: True,
+) -> ta.Mapping[ta.Callable, cf.Future]:
+    for fn, deps in dependency_sets_by_fn.items():
+        for dep in deps:
+            if dep == fn:
+                raise ValueError(fn)
+            if dep not in dependency_sets_by_fn:
+                raise KeyError(dep)
+            if fn in dependency_sets_by_fn[dep]:
+                raise Exception(f'Cyclic dependencies: {fn} <-> {dep}', fn, dep)
+
+    dependent_sets_by_fn: ta.Dict[ta.Callable, ta.Set[ta.Callable]] = {fn: set() for fn in dependency_sets_by_fn}
+    for fn, deps in dependency_sets_by_fn.items():
+        for dep in deps:
+            dependent_sets_by_fn[dep].add(fn)
+    remaining_dep_sets_by_fn = {
+        fn: set(dependencies) for fn, dependencies in dependency_sets_by_fn.items()
+    }
+    root_fns = {fn for fn, deps in remaining_dep_sets_by_fn.items() if not deps}
+    fns_by_fut = {fut: fn for fn in root_fns for fut in [executor.submit(fn)] if fut is not None}
+
+    def cancel():
+        for cancel_fut in fns_by_fut:
+            cancel_fut.cancel()
+
+    start = time.time()
+    not_done = set(fns_by_fut.keys())
+    while not_done and tick_fn():
+        done, not_done = cf.wait(not_done, timeout=tick_interval_s, return_when=cf.FIRST_COMPLETED)
+        not_done = set(not_done)
+
+        for fut in done:
+            if fut.exception():
+                cancel()
+                raise FutureException(fut) from fut.exception()
+
+            fn = fns_by_fut[fut]
+            for dependent_fn in dependent_sets_by_fn.get(fn, set()):
+                remaining_deps = remaining_dep_sets_by_fn[dependent_fn]
+                remaining_deps.remove(fn)
+                if not remaining_deps:
+                    downstream_fut = executor.submit(dependent_fn)
+                    if downstream_fut is not None:
+                        fns_by_fut[downstream_fut] = dependent_fn
+                        not_done.add(downstream_fut)
+
+        if time.time() >= (start + timeout_s):
+            cancel()
+            raise FutureTimeoutException
+
+    remaining_fns = {fn: deps for fn, deps in remaining_dep_sets_by_fn.items() if deps}
+    if remaining_fns:
+        raise Exception(f"Unfinished fns: {remaining_fns}", remaining_fns)
+
+    futs_by_fn = {fn: fut for fut, fn in fns_by_fut.items()}
+    return futs_by_fn
+
+
+##
+
+
+class ImmediateExecutor(cf.Executor):
+
+    def __init__(self, *, immediate_exceptions: bool = False) -> None:
+        super().__init__()
+        self._immediate_exceptions = immediate_exceptions
+
+    def submit(self, fn, *args, **kwargs):
+        future: ta.Any = cf.Future()
+        try:
+            result = fn(*args, **kwargs)
+            future.set_result(result)
+        except Exception as e:
+            if self._immediate_exceptions:
+                raise
+            future.set_exception(e)
+        return future
+
+
+@contextlib.contextmanager
+def new_thread_or_immediate_executor(
+        max_workers: int | None = None,
+        *,
+        immediate_exceptions: bool = False,
+        thread_name_prefix: str = '',
+        **kwargs: ta.Any,
+) -> ta.Generator[cf.Executor, None, None]:
+    if max_workers == 0:
+        yield ImmediateExecutor(
+            immediate_exceptions=immediate_exceptions,
+        )
+    else:
+        with cf.ThreadPoolExecutor(
+                thread_name_prefix=thread_name_prefix,
+                **kwargs,
+        ) as exe:
+            yield exe

omlish/asyncs/trio.py

@@ -0,0 +1,11 @@
+"""
+lookit:
+ - https://github.com/oremanj/trio-monitor
+ - https://github.com/python-trio/trio-monitor/tree/master
+ - https://github.com/python-trio/triopg
+ - https://github.com/python-trio/trio-mysql
+ - https://github.com/goodboy/tractor
+ - https://github.com/groove-x/trio-util
+ - https://github.com/oremanj/tricycle/tree/master
+ - https://github.com/linkdd/triotp
+"""