omlish 0.0.0.dev17__tar.gz → 0.0.0.dev19__tar.gz

{omlish-0.0.0.dev17/omlish.egg-info → omlish-0.0.0.dev19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omlish
-Version: 0.0.0.dev17
+Version: 0.0.0.dev19
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause

{omlish-0.0.0.dev17 → omlish-0.0.0.dev19}/omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev17'
-__revision__ = 'c238de47faafaf823b552b9e5cf22e47f5c9cd2b'
+__version__ = '0.0.0.dev19'
+__revision__ = '509bb451054d2d955e55cbcf1f271a9166a904eb'
 
 
 #

{omlish-0.0.0.dev17 → omlish-0.0.0.dev19}/omlish/asyncs/anyio.py

@@ -37,6 +37,12 @@ from .. import lang
 
 T = ta.TypeVar('T')
 
+MemoryObjectReceiveStream: ta.TypeAlias = anyio.streams.memory.MemoryObjectReceiveStream
+MemoryObjectSendStream: ta.TypeAlias = anyio.streams.memory.MemoryObjectSendStream
+
+StapledByteStream: ta.TypeAlias = anyio.streams.stapled.StapledByteStream
+StapledObjectStream: ta.TypeAlias = anyio.streams.stapled.StapledObjectStream
+
 
 ##
 
@@ -130,39 +136,43 @@ def get_current_backend_task() -> BackendTask | None:
 def split_memory_object_streams(
         *args: anyio.create_memory_object_stream[T],
 ) -> tuple[
-    anyio.streams.memory.MemoryObjectSendStream[T],
-    anyio.streams.memory.MemoryObjectReceiveStream[T],
+    MemoryObjectSendStream[T],
+    MemoryObjectReceiveStream[T],
 ]:
     [tup] = args
     return tup
 
 
+def create_stapled_memory_object_stream(max_buffer_size: float = 0) -> StapledObjectStream:
+    return StapledObjectStream(*anyio.create_memory_object_stream(max_buffer_size))
+
+
 # FIXME: https://github.com/python/mypy/issues/15238
 # FIXME: https://youtrack.jetbrains.com/issues?q=tag:%20%7BPEP%20695%7D
 def create_memory_object_stream[T](max_buffer_size: float = 0) -> tuple[
-    anyio.streams.memory.MemoryObjectSendStream[T],
-    anyio.streams.memory.MemoryObjectReceiveStream[T],
+    MemoryObjectSendStream[T],
+    MemoryObjectReceiveStream[T],
 ]:
     return anyio.create_memory_object_stream[T](max_buffer_size)  # noqa
 
 
 def staple_memory_object_stream(
         *args: anyio.create_memory_object_stream[T],
-) -> anyio.streams.stapled.StapledObjectStream[T]:
+) -> StapledObjectStream[T]:
     send, receive = args
-    return anyio.streams.stapled.StapledObjectStream(
-        check.isinstance(send, anyio.streams.memory.MemoryObjectSendStream),  # type: ignore
-        check.isinstance(receive, anyio.streams.memory.MemoryObjectReceiveStream),  # type: ignore
+    return StapledObjectStream(
+        check.isinstance(send, MemoryObjectSendStream),  # type: ignore
+        check.isinstance(receive, MemoryObjectReceiveStream),  # type: ignore
     )
 
 
 # FIXME: https://github.com/python/mypy/issues/15238
 # FIXME: https://youtrack.jetbrains.com/issues?q=tag:%20%7BPEP%20695%7D
-def staple_memory_object_stream2[T](max_buffer_size: float = 0) -> anyio.streams.stapled.StapledObjectStream[T]:
+def staple_memory_object_stream2[T](max_buffer_size: float = 0) -> StapledObjectStream[T]:
     send, receive = anyio.create_memory_object_stream[T](max_buffer_size)
-    return anyio.streams.stapled.StapledObjectStream(
-        check.isinstance(send, anyio.streams.memory.MemoryObjectSendStream),  # type: ignore
-        check.isinstance(receive, anyio.streams.memory.MemoryObjectReceiveStream),  # type: ignore
+    return StapledObjectStream(
+        check.isinstance(send, MemoryObjectSendStream),  # type: ignore
+        check.isinstance(receive, MemoryObjectReceiveStream),  # type: ignore
     )
 
 

{omlish-0.0.0.dev17 → omlish-0.0.0.dev19}/omlish/asyncs/flavors.py

@@ -1,8 +1,8 @@
 """
+Tools for working with the different async loop implementations - currently anyio, asyncio, trio.
+
 TODO:
- - 'get current'? -> sniffio..
  - mark whole class / module?
- - sync/greenlet bridge
 """
 import abc
 import dataclasses as dc
@@ -73,11 +73,13 @@ def _get_module_flavor(p: str) -> Flavor | None:
         return _MODULE_FLAVOR_CACHE[p]
     except KeyError:
         pass
+
     pf: Flavor | None = None
     for cp, cf in PACKAGE_FLAVORS.items():
         if p.startswith(cp) and (len(cp) == len(p) or p[len(cp)] == '.'):
             pf = cf
             break
+
     _MODULE_FLAVOR_CACHE[p] = pf
     return pf
 

omlish-0.0.0.dev19/omlish/bootstrap/__init__.py

@@ -0,0 +1,3 @@
+from .harness import (  # noqa
+    bootstrap,
+)

omlish-0.0.0.dev19/omlish/bootstrap/__main__.py

@@ -0,0 +1,4 @@
+if __name__ == '__main__':
+    from .main import _main
+
+    _main()

omlish-0.0.0.dev19/omlish/bootstrap/base.py

@@ -0,0 +1,38 @@
+import abc
+import dataclasses as dc
+import typing as ta
+
+
+BootstrapConfigT = ta.TypeVar('BootstrapConfigT', bound='Bootstrap.Config')
+
+
+##
+
+
+class Bootstrap(abc.ABC, ta.Generic[BootstrapConfigT]):
+    @dc.dataclass(frozen=True)
+    class Config(abc.ABC):  # noqa
+        pass
+
+    def __init__(self, config: BootstrapConfigT) -> None:
+        super().__init__()
+        self._config = config
+
+    def __init_subclass__(cls, **kwargs: ta.Any) -> None:
+        super().__init_subclass__(**kwargs)
+        if not cls.__name__.endswith('Bootstrap'):
+            raise NameError(cls)
+        if abc.ABC not in cls.__bases__ and not issubclass(cls.__dict__['Config'], Bootstrap.Config):
+            raise TypeError(cls)
+
+
+class SimpleBootstrap(Bootstrap[BootstrapConfigT], abc.ABC):
+    @abc.abstractmethod
+    def run(self) -> None:
+        raise NotImplementedError
+
+
+class ContextBootstrap(Bootstrap[BootstrapConfigT], abc.ABC):
+    @abc.abstractmethod
+    def enter(self) -> ta.ContextManager[None]:
+        raise NotImplementedError

omlish-0.0.0.dev19/omlish/bootstrap/diag.py

@@ -0,0 +1,128 @@
+# ruff: noqa: UP007
+import contextlib
+import dataclasses as dc
+import signal
+import sys
+import typing as ta
+
+from .. import lang
+from .base import Bootstrap
+from .base import ContextBootstrap
+
+
+if ta.TYPE_CHECKING:
+    import cProfile  # noqa
+    import pstats
+
+    from ..diag import threads as diagt
+
+else:
+    cProfile = lang.proxy_import('cProfile')  # noqa
+    pstats = lang.proxy_import('pstats')
+
+    diagt = lang.proxy_import('..diag.threads', __package__)
+
+
+##
+
+
+class ProfilingBootstrap(ContextBootstrap['ProfilingBootstrap.Config']):
+    @dc.dataclass(frozen=True)
+    class Config(Bootstrap.Config):
+        enable: bool = False
+        builtins: bool = True
+
+        outfile: ta.Optional[str] = None
+
+        print: bool = False
+        sort: str = 'cumtime'
+        topn: int = 100
+
+    @contextlib.contextmanager
+    def enter(self) -> ta.Iterator[None]:
+        if not self._config.enable:
+            yield
+            return
+
+        prof = cProfile.Profile()
+        prof.enable()
+        try:
+            yield
+
+        finally:
+            prof.disable()
+            prof.create_stats()
+
+            if self._config.print:
+                pstats.Stats(prof) \
+                    .strip_dirs() \
+                    .sort_stats(self._config.sort) \
+                    .print_stats(self._config.topn)
+
+            if self._config.outfile is not None:
+                prof.dump_stats(self._config.outfile)
+
+
+##
+
+
+class ThreadDumpBootstrap(ContextBootstrap['ThreadDumpBootstrap.Config']):
+    @dc.dataclass(frozen=True)
+    class Config(Bootstrap.Config):
+        interval_s: ta.Optional[float] = None
+
+        on_sigquit: bool = False
+
+    @contextlib.contextmanager
+    def enter(self) -> ta.Iterator[None]:
+        if self._config.interval_s:
+            tdt = diagt.create_thread_dump_thread(
+                interval_s=self._config.interval_s,
+                start=True,
+            )
+        else:
+            tdt = None
+
+        if self._config.on_sigquit:
+            dump_threads_str = diagt.dump_threads_str
+
+            def handler(signum, frame):
+                print(dump_threads_str(), file=sys.stderr)
+
+            prev_sq = lang.just(signal.signal(signal.SIGQUIT, handler))
+        else:
+            prev_sq = lang.empty()
+
+        try:
+            yield
+
+        finally:
+            if tdt is not None:
+                tdt.stop_nowait()
+
+            if prev_sq.present:
+                signal.signal(signal.SIGQUIT, prev_sq.must())
+
+
+##
+
+
+class TimebombBootstrap(ContextBootstrap['TimebombBootstrap.Config']):
+    @dc.dataclass(frozen=True)
+    class Config(Bootstrap.Config):
+        delay_s: ta.Optional[float] = None
+
+    @contextlib.contextmanager
+    def enter(self) -> ta.Iterator[None]:
+        if not self._config.delay_s:
+            yield
+            return
+
+        tbt = diagt.create_timebomb_thread(
+            self._config.delay_s,
+            start=True,
+        )
+        try:
+            yield
+        finally:
+            tbt.stop_nowait()

omlish-0.0.0.dev19/omlish/bootstrap/harness.py

@@ -0,0 +1,81 @@
+"""
+TODO:
+ - more logging options
+ - a more powerful interface would be run_fn_with_bootstrap..
+ - ./python -m gprof2dot -f pstats prof.pstats | dot -Tpdf -o prof.pstats.pdf && open prof.pstats.pdf
+ - multiprocess profiling - afterfork, suffix with pid
+
+TODO diag:
+ - yappi
+ - stackscope
+ - https://github.com/pythonspeed/filprofiler
+ - https://pypi.org/project/guppy3/
+ - https://pypi.org/project/memory-profiler/
+ - https://pypi.org/project/Pympler/
+
+TODO new items:
+ - pydevd connect-back
+ - debugging / pdb
+ - repl server
+ - packaging fixups
+ - daemonize ( https://github.com/thesharp/daemonize/blob/master/daemonize.py )
+"""
+# ruff: noqa: UP006
+import contextlib
+import typing as ta
+
+from .. import lang
+from . import diag  # noqa
+from . import sys  # noqa
+from .base import Bootstrap
+from .base import ContextBootstrap
+from .base import SimpleBootstrap
+
+
+##
+
+
+BOOTSTRAP_TYPES_BY_NAME: ta.Mapping[str, ta.Type[Bootstrap]] = {  # noqa
+    lang.snake_case(cls.__name__[:-len('Bootstrap')]): cls
+    for cls in lang.deep_subclasses(Bootstrap)
+    if not lang.is_abstract_class(cls)
+}
+
+BOOTSTRAP_TYPES_BY_CONFIG_TYPE: ta.Mapping[ta.Type[Bootstrap.Config], ta.Type[Bootstrap]] = {
+    cls.Config: cls
+    for cls in BOOTSTRAP_TYPES_BY_NAME.values()
+}
+
+
+##
+
+
+class BootstrapHarness:
+    def __init__(self, lst: ta.Sequence[Bootstrap]) -> None:
+        super().__init__()
+        self._lst = lst
+
+    @contextlib.contextmanager
+    def __call__(self) -> ta.Iterator[None]:
+        with contextlib.ExitStack() as es:
+            for c in self._lst:
+                if isinstance(c, SimpleBootstrap):
+                    c.run()
+                elif isinstance(c, ContextBootstrap):
+                    es.enter_context(c.enter())
+                else:
+                    raise TypeError(c)
+
+            yield
+
+
+##
+
+
+@contextlib.contextmanager
+def bootstrap(*cfgs: Bootstrap.Config) -> ta.Iterator[None]:
+    with BootstrapHarness([
+        BOOTSTRAP_TYPES_BY_CONFIG_TYPE[type(c)](c)
+        for c in cfgs
+    ])():
+        yield

omlish-0.0.0.dev19/omlish/bootstrap/main.py

@@ -0,0 +1,183 @@
+# ruff: noqa: UP006 UP007
+import argparse
+import dataclasses as dc
+import io
+import itertools
+import sys
+import typing as ta
+
+from .. import lang
+from .base import Bootstrap
+from .harness import BOOTSTRAP_TYPES_BY_NAME
+from .harness import bootstrap
+
+
+if ta.TYPE_CHECKING:
+    import runpy
+
+else:
+    runpy = lang.proxy_import('runpy')
+
+
+##
+
+
+class _OrderedArgsAction(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        if 'ordered_args' not in namespace:
+            setattr(namespace, 'ordered_args', [])
+        if self.const is not None:
+            value = self.const
+        else:
+            value = values
+        namespace.ordered_args.append((self.dest, value))
+
+
+def _or_opt(ty):
+    return (ty, ta.Optional[ty])
+
+
+def _int_or_str(v):
+    try:
+        return int(v)
+    except ValueError:
+        return v
+
+
+def _add_arguments(parser: argparse.ArgumentParser) -> None:
+    # ta.Optional[ta.Mapping[str, ta.Tuple[ta.Optional[int], ta.Optional[int]]]]
+
+    for cname, cls in BOOTSTRAP_TYPES_BY_NAME.items():
+        for fld in dc.fields(cls.Config):
+            aname = f'--{cname}:{fld.name}'
+            kw: ta.Dict[str, ta.Any] = {}
+
+            if fld.type in _or_opt(str):
+                pass
+            elif fld.type in _or_opt(bool):
+                kw.update(const=True, nargs=0)
+            elif fld.type in _or_opt(int):
+                kw.update(type=int)
+            elif fld.type in _or_opt(float):
+                kw.update(type=float)
+            elif fld.type in _or_opt(ta.Union[int, str]):
+                kw.update(type=_int_or_str)
+
+            elif fld.type in (
+                    *_or_opt(ta.Sequence[str]),
+                    *_or_opt(ta.Mapping[str, ta.Optional[str]]),
+                    *_or_opt(ta.Mapping[int, ta.Union[int, str, None]]),
+                    *_or_opt(ta.Mapping[str, ta.Tuple[ta.Optional[int], ta.Optional[int]]]),
+            ):
+                if aname[-1] != 's':
+                    raise NameError(aname)
+                aname = aname[:-1]
+
+            else:
+                raise TypeError(fld)
+
+            parser.add_argument(aname, action=_OrderedArgsAction, **kw)
+
+
+def _process_arguments(args: ta.Any) -> ta.Sequence[Bootstrap.Config]:
+    if not (oa := getattr(args, 'ordered_args', None)):
+        return []
+
+    cfgs: ta.List[Bootstrap.Config] = []
+
+    for cname, cargs in [
+        (n, list(g))
+        for n, g in
+        itertools.groupby(oa, key=lambda s: s[0].partition(':')[0])
+    ]:
+        ccls = BOOTSTRAP_TYPES_BY_NAME[cname].Config
+        flds = {f.name: f for f in dc.fields(ccls)}
+
+        kw: ta.Dict[str, ta.Any] = {}
+        for aname, aval in cargs:
+            k = aname.partition(':')[2]
+
+            if k not in flds:
+                k += 's'
+                fld = flds[k]
+
+                if fld.type in _or_opt(ta.Sequence[str]):
+                    kw.setdefault(k, []).append(aval)
+
+                elif fld.type in _or_opt(ta.Mapping[str, ta.Optional[str]]):
+                    if '=' not in aval:
+                        kw.setdefault(k, {})[aval] = None
+                    else:
+                        ek, _, ev = aval.partition('=')
+                        kw.setdefault(k, {})[ek] = ev
+
+                elif fld.type in _or_opt(ta.Mapping[int, ta.Union[int, str, None]]):
+                    fk, _, fv = aval.partition('=')
+                    if not fv:
+                        kw.setdefault(k, {})[int(fk)] = None
+                    else:
+                        kw.setdefault(k, {})[int(fk)] = _int_or_str(fv)
+
+                elif fld.type in _or_opt(ta.Mapping[str, ta.Tuple[ta.Optional[int], ta.Optional[int]]]):
+                    fk, _, fv = aval.partition('=')
+                    if ',' in fv:
+                        tl, tr = fv.split(',')
+                    else:
+                        tl, tr = None, None
+                    kw.setdefault(k, {})[fk] = (_int_or_str(tl) if tl else None, _int_or_str(tr) if tr else None)
+
+                else:
+                    raise TypeError(fld)
+
+            else:
+                kw[k] = aval
+
+        cfg = ccls(**kw)
+        cfgs.append(cfg)
+
+    return cfgs
+
+
+##
+
+
+def _main() -> int:
+    parser = argparse.ArgumentParser()
+
+    _add_arguments(parser)
+
+    parser.add_argument('-m', '--module', action='store_true')
+    parser.add_argument('target')
+    parser.add_argument('args', nargs=argparse.REMAINDER)
+
+    args = parser.parse_args()
+    cfgs = _process_arguments(args)
+
+    with bootstrap(*cfgs):
+        tgt = args.target
+
+        if args.module:
+            sys.argv = [tgt, *(args.args or ())]
+            runpy._run_module_as_main(tgt)  # type: ignore  # noqa
+
+        else:
+            with io.open_code(tgt) as fp:
+                src = fp.read()
+
+            ns = dict(
+                __name__='__main__',
+                __file__=tgt,
+                __builtins__=__builtins__,
+                __spec__=None,
+            )
+
+            import __main__  # noqa
+            __main__.__dict__.clear()
+            __main__.__dict__.update(ns)
+            exec(compile(src, tgt, 'exec'), __main__.__dict__, __main__.__dict__)
+
+        return 0
+
+
+if __name__ == '__main__':
+    sys.exit(_main())