omlish 0.0.0.dev18__py3-none-any.whl → 0.0.0.dev20__py3-none-any.whl

omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev18'
-__revision__ = '534f2a8b308f6514a06a41bba629c71a05497327'
+__version__ = '0.0.0.dev20'
+__revision__ = '56a3421b375112818e306f2c17d467c7aac1d26b'
 
 
 #
@@ -88,7 +88,7 @@ class Project(ProjectBase):
         'sqlx': [
             'sqlean.py ~= 3.45; python_version < "3.13"',
 
-            'duckdb ~= 1.0',
+            'duckdb ~= 1.1',
         ],
 
         'testing': [

omlish/asyncs/anyio.py

@@ -23,6 +23,7 @@ async def killer(shutdown: anyio.Event, sleep_s: float) -> None:
     shutdown.set()
 
 """  # noqa
+import dataclasses as dc
 import signal
 import typing as ta
 
@@ -44,6 +45,12 @@ StapledByteStream: ta.TypeAlias = anyio.streams.stapled.StapledByteStream
 StapledObjectStream: ta.TypeAlias = anyio.streams.stapled.StapledObjectStream
 
 
+@dc.dataclass(eq=False)
+class MemoryStapledObjectStream(StapledObjectStream[T]):
+    send_stream: MemoryObjectSendStream[T]
+    receive_stream: MemoryObjectReceiveStream[T]
+
+
 ##
 
 
@@ -143,8 +150,8 @@ def split_memory_object_streams(
     return tup
 
 
-def create_stapled_memory_object_stream(max_buffer_size: float = 0) -> StapledObjectStream:
-    return StapledObjectStream(*anyio.create_memory_object_stream(max_buffer_size))
+def create_stapled_memory_object_stream(max_buffer_size: float = 0) -> MemoryStapledObjectStream:
+    return MemoryStapledObjectStream(*anyio.create_memory_object_stream(max_buffer_size))
 
 
 # FIXME: https://github.com/python/mypy/issues/15238
@@ -158,9 +165,9 @@ def create_memory_object_stream[T](max_buffer_size: float = 0) -> tuple[
 
 def staple_memory_object_stream(
         *args: anyio.create_memory_object_stream[T],
-) -> StapledObjectStream[T]:
+) -> MemoryStapledObjectStream[T]:
     send, receive = args
-    return StapledObjectStream(
+    return MemoryStapledObjectStream(
         check.isinstance(send, MemoryObjectSendStream),  # type: ignore
         check.isinstance(receive, MemoryObjectReceiveStream),  # type: ignore
     )
@@ -168,9 +175,9 @@ def staple_memory_object_stream(
 
 # FIXME: https://github.com/python/mypy/issues/15238
 # FIXME: https://youtrack.jetbrains.com/issues?q=tag:%20%7BPEP%20695%7D
-def staple_memory_object_stream2[T](max_buffer_size: float = 0) -> StapledObjectStream[T]:
+def staple_memory_object_stream2[T](max_buffer_size: float = 0) -> MemoryStapledObjectStream[T]:
     send, receive = anyio.create_memory_object_stream[T](max_buffer_size)
-    return StapledObjectStream(
+    return MemoryStapledObjectStream(
         check.isinstance(send, MemoryObjectSendStream),  # type: ignore
         check.isinstance(receive, MemoryObjectReceiveStream),  # type: ignore
     )

omlish/bootstrap/__init__.py

@@ -0,0 +1,3 @@
+from .harness import (  # noqa
+    bootstrap,
+)

omlish/bootstrap/__main__.py

@@ -0,0 +1,4 @@
+if __name__ == '__main__':
+    from .main import _main
+
+    _main()

omlish/bootstrap/base.py

@@ -0,0 +1,38 @@
+import abc
+import dataclasses as dc
+import typing as ta
+
+
+BootstrapConfigT = ta.TypeVar('BootstrapConfigT', bound='Bootstrap.Config')
+
+
+##
+
+
+class Bootstrap(abc.ABC, ta.Generic[BootstrapConfigT]):
+    @dc.dataclass(frozen=True)
+    class Config(abc.ABC):  # noqa
+        pass
+
+    def __init__(self, config: BootstrapConfigT) -> None:
+        super().__init__()
+        self._config = config
+
+    def __init_subclass__(cls, **kwargs: ta.Any) -> None:
+        super().__init_subclass__(**kwargs)
+        if not cls.__name__.endswith('Bootstrap'):
+            raise NameError(cls)
+        if abc.ABC not in cls.__bases__ and not issubclass(cls.__dict__['Config'], Bootstrap.Config):
+            raise TypeError(cls)
+
+
+class SimpleBootstrap(Bootstrap[BootstrapConfigT], abc.ABC):
+    @abc.abstractmethod
+    def run(self) -> None:
+        raise NotImplementedError
+
+
+class ContextBootstrap(Bootstrap[BootstrapConfigT], abc.ABC):
+    @abc.abstractmethod
+    def enter(self) -> ta.ContextManager[None]:
+        raise NotImplementedError

omlish/bootstrap/diag.py

@@ -0,0 +1,128 @@
+# ruff: noqa: UP007
+import contextlib
+import dataclasses as dc
+import signal
+import sys
+import typing as ta
+
+from .. import lang
+from .base import Bootstrap
+from .base import ContextBootstrap
+
+
+if ta.TYPE_CHECKING:
+    import cProfile  # noqa
+    import pstats
+
+    from ..diag import threads as diagt
+
+else:
+    cProfile = lang.proxy_import('cProfile')  # noqa
+    pstats = lang.proxy_import('pstats')
+
+    diagt = lang.proxy_import('..diag.threads', __package__)
+
+
+##
+
+
+class ProfilingBootstrap(ContextBootstrap['ProfilingBootstrap.Config']):
+    @dc.dataclass(frozen=True)
+    class Config(Bootstrap.Config):
+        enable: bool = False
+        builtins: bool = True
+
+        outfile: ta.Optional[str] = None
+
+        print: bool = False
+        sort: str = 'cumtime'
+        topn: int = 100
+
+    @contextlib.contextmanager
+    def enter(self) -> ta.Iterator[None]:
+        if not self._config.enable:
+            yield
+            return
+
+        prof = cProfile.Profile()
+        prof.enable()
+        try:
+            yield
+
+        finally:
+            prof.disable()
+            prof.create_stats()
+
+            if self._config.print:
+                pstats.Stats(prof) \
+                    .strip_dirs() \
+                    .sort_stats(self._config.sort) \
+                    .print_stats(self._config.topn)
+
+            if self._config.outfile is not None:
+                prof.dump_stats(self._config.outfile)
+
+
+##
+
+
+class ThreadDumpBootstrap(ContextBootstrap['ThreadDumpBootstrap.Config']):
+    @dc.dataclass(frozen=True)
+    class Config(Bootstrap.Config):
+        interval_s: ta.Optional[float] = None
+
+        on_sigquit: bool = False
+
+    @contextlib.contextmanager
+    def enter(self) -> ta.Iterator[None]:
+        if self._config.interval_s:
+            tdt = diagt.create_thread_dump_thread(
+                interval_s=self._config.interval_s,
+                start=True,
+            )
+        else:
+            tdt = None
+
+        if self._config.on_sigquit:
+            dump_threads_str = diagt.dump_threads_str
+
+            def handler(signum, frame):
+                print(dump_threads_str(), file=sys.stderr)
+
+            prev_sq = lang.just(signal.signal(signal.SIGQUIT, handler))
+        else:
+            prev_sq = lang.empty()
+
+        try:
+            yield
+
+        finally:
+            if tdt is not None:
+                tdt.stop_nowait()
+
+            if prev_sq.present:
+                signal.signal(signal.SIGQUIT, prev_sq.must())
+
+
+##
+
+
+class TimebombBootstrap(ContextBootstrap['TimebombBootstrap.Config']):
+    @dc.dataclass(frozen=True)
+    class Config(Bootstrap.Config):
+        delay_s: ta.Optional[float] = None
+
+    @contextlib.contextmanager
+    def enter(self) -> ta.Iterator[None]:
+        if not self._config.delay_s:
+            yield
+            return
+
+        tbt = diagt.create_timebomb_thread(
+            self._config.delay_s,
+            start=True,
+        )
+        try:
+            yield
+        finally:
+            tbt.stop_nowait()

omlish/bootstrap/harness.py

@@ -0,0 +1,81 @@
+"""
+TODO:
+ - more logging options
+ - a more powerful interface would be run_fn_with_bootstrap..
+ - ./python -m gprof2dot -f pstats prof.pstats | dot -Tpdf -o prof.pstats.pdf && open prof.pstats.pdf
+ - multiprocess profiling - afterfork, suffix with pid
+
+TODO diag:
+ - yappi
+ - stackscope
+ - https://github.com/pythonspeed/filprofiler
+ - https://pypi.org/project/guppy3/
+ - https://pypi.org/project/memory-profiler/
+ - https://pypi.org/project/Pympler/
+
+TODO new items:
+ - pydevd connect-back
+ - debugging / pdb
+ - repl server
+ - packaging fixups
+ - daemonize ( https://github.com/thesharp/daemonize/blob/master/daemonize.py )
+"""
+# ruff: noqa: UP006
+import contextlib
+import typing as ta
+
+from .. import lang
+from . import diag  # noqa
+from . import sys  # noqa
+from .base import Bootstrap
+from .base import ContextBootstrap
+from .base import SimpleBootstrap
+
+
+##
+
+
+BOOTSTRAP_TYPES_BY_NAME: ta.Mapping[str, ta.Type[Bootstrap]] = {  # noqa
+    lang.snake_case(cls.__name__[:-len('Bootstrap')]): cls
+    for cls in lang.deep_subclasses(Bootstrap)
+    if not lang.is_abstract_class(cls)
+}
+
+BOOTSTRAP_TYPES_BY_CONFIG_TYPE: ta.Mapping[ta.Type[Bootstrap.Config], ta.Type[Bootstrap]] = {
+    cls.Config: cls
+    for cls in BOOTSTRAP_TYPES_BY_NAME.values()
+}
+
+
+##
+
+
+class BootstrapHarness:
+    def __init__(self, lst: ta.Sequence[Bootstrap]) -> None:
+        super().__init__()
+        self._lst = lst
+
+    @contextlib.contextmanager
+    def __call__(self) -> ta.Iterator[None]:
+        with contextlib.ExitStack() as es:
+            for c in self._lst:
+                if isinstance(c, SimpleBootstrap):
+                    c.run()
+                elif isinstance(c, ContextBootstrap):
+                    es.enter_context(c.enter())
+                else:
+                    raise TypeError(c)
+
+            yield
+
+
+##
+
+
+@contextlib.contextmanager
+def bootstrap(*cfgs: Bootstrap.Config) -> ta.Iterator[None]:
+    with BootstrapHarness([
+        BOOTSTRAP_TYPES_BY_CONFIG_TYPE[type(c)](c)
+        for c in cfgs
+    ])():
+        yield

omlish/bootstrap/main.py

@@ -0,0 +1,183 @@
+# ruff: noqa: UP006 UP007
+import argparse
+import dataclasses as dc
+import io
+import itertools
+import sys
+import typing as ta
+
+from .. import lang
+from .base import Bootstrap
+from .harness import BOOTSTRAP_TYPES_BY_NAME
+from .harness import bootstrap
+
+
+if ta.TYPE_CHECKING:
+    import runpy
+
+else:
+    runpy = lang.proxy_import('runpy')
+
+
+##
+
+
+class _OrderedArgsAction(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        if 'ordered_args' not in namespace:
+            setattr(namespace, 'ordered_args', [])
+        if self.const is not None:
+            value = self.const
+        else:
+            value = values
+        namespace.ordered_args.append((self.dest, value))
+
+
+def _or_opt(ty):
+    return (ty, ta.Optional[ty])
+
+
+def _int_or_str(v):
+    try:
+        return int(v)
+    except ValueError:
+        return v
+
+
+def _add_arguments(parser: argparse.ArgumentParser) -> None:
+    # ta.Optional[ta.Mapping[str, ta.Tuple[ta.Optional[int], ta.Optional[int]]]]
+
+    for cname, cls in BOOTSTRAP_TYPES_BY_NAME.items():
+        for fld in dc.fields(cls.Config):
+            aname = f'--{cname}:{fld.name}'
+            kw: ta.Dict[str, ta.Any] = {}
+
+            if fld.type in _or_opt(str):
+                pass
+            elif fld.type in _or_opt(bool):
+                kw.update(const=True, nargs=0)
+            elif fld.type in _or_opt(int):
+                kw.update(type=int)
+            elif fld.type in _or_opt(float):
+                kw.update(type=float)
+            elif fld.type in _or_opt(ta.Union[int, str]):
+                kw.update(type=_int_or_str)
+
+            elif fld.type in (
+                    *_or_opt(ta.Sequence[str]),
+                    *_or_opt(ta.Mapping[str, ta.Optional[str]]),
+                    *_or_opt(ta.Mapping[int, ta.Union[int, str, None]]),
+                    *_or_opt(ta.Mapping[str, ta.Tuple[ta.Optional[int], ta.Optional[int]]]),
+            ):
+                if aname[-1] != 's':
+                    raise NameError(aname)
+                aname = aname[:-1]
+
+            else:
+                raise TypeError(fld)
+
+            parser.add_argument(aname, action=_OrderedArgsAction, **kw)
+
+
+def _process_arguments(args: ta.Any) -> ta.Sequence[Bootstrap.Config]:
+    if not (oa := getattr(args, 'ordered_args', None)):
+        return []
+
+    cfgs: ta.List[Bootstrap.Config] = []
+
+    for cname, cargs in [
+        (n, list(g))
+        for n, g in
+        itertools.groupby(oa, key=lambda s: s[0].partition(':')[0])
+    ]:
+        ccls = BOOTSTRAP_TYPES_BY_NAME[cname].Config
+        flds = {f.name: f for f in dc.fields(ccls)}
+
+        kw: ta.Dict[str, ta.Any] = {}
+        for aname, aval in cargs:
+            k = aname.partition(':')[2]
+
+            if k not in flds:
+                k += 's'
+                fld = flds[k]
+
+                if fld.type in _or_opt(ta.Sequence[str]):
+                    kw.setdefault(k, []).append(aval)
+
+                elif fld.type in _or_opt(ta.Mapping[str, ta.Optional[str]]):
+                    if '=' not in aval:
+                        kw.setdefault(k, {})[aval] = None
+                    else:
+                        ek, _, ev = aval.partition('=')
+                        kw.setdefault(k, {})[ek] = ev
+
+                elif fld.type in _or_opt(ta.Mapping[int, ta.Union[int, str, None]]):
+                    fk, _, fv = aval.partition('=')
+                    if not fv:
+                        kw.setdefault(k, {})[int(fk)] = None
+                    else:
+                        kw.setdefault(k, {})[int(fk)] = _int_or_str(fv)
+
+                elif fld.type in _or_opt(ta.Mapping[str, ta.Tuple[ta.Optional[int], ta.Optional[int]]]):
+                    fk, _, fv = aval.partition('=')
+                    if ',' in fv:
+                        tl, tr = fv.split(',')
+                    else:
+                        tl, tr = None, None
+                    kw.setdefault(k, {})[fk] = (_int_or_str(tl) if tl else None, _int_or_str(tr) if tr else None)
+
+                else:
+                    raise TypeError(fld)
+
+            else:
+                kw[k] = aval
+
+        cfg = ccls(**kw)
+        cfgs.append(cfg)
+
+    return cfgs
+
+
+##
+
+
+def _main() -> int:
+    parser = argparse.ArgumentParser()
+
+    _add_arguments(parser)
+
+    parser.add_argument('-m', '--module', action='store_true')
+    parser.add_argument('target')
+    parser.add_argument('args', nargs=argparse.REMAINDER)
+
+    args = parser.parse_args()
+    cfgs = _process_arguments(args)
+
+    with bootstrap(*cfgs):
+        tgt = args.target
+
+        if args.module:
+            sys.argv = [tgt, *(args.args or ())]
+            runpy._run_module_as_main(tgt)  # type: ignore  # noqa
+
+        else:
+            with io.open_code(tgt) as fp:
+                src = fp.read()
+
+            ns = dict(
+                __name__='__main__',
+                __file__=tgt,
+                __builtins__=__builtins__,
+                __spec__=None,
+            )
+
+            import __main__  # noqa
+            __main__.__dict__.clear()
+            __main__.__dict__.update(ns)
+            exec(compile(src, tgt, 'exec'), __main__.__dict__, __main__.__dict__)
+
+        return 0
+
+
+if __name__ == '__main__':
+    sys.exit(_main())