omlish 0.0.0.dev5__py3-none-any.whl → 0.0.0.dev7__py3-none-any.whl

omlish/{testing → diag}/pydevd.py

@@ -32,6 +32,7 @@ import os
 import sys
 import tempfile
 import textwrap
+import threading
 import types
 import typing as ta
 
@@ -243,3 +244,37 @@ def maybe_reexec(
         args = [args[0], bootstrap_path]
 
     os.execvp(sys.executable, args)
+
+
+def debug_unhandled_exception(exc_info: ta.Any = None) -> None:
+    if exc_info is None:
+        exc_info = sys.exc_info()
+
+    try:
+        import pydevd
+        from pydevd import pydevd_tracing
+
+    except ImportError:
+        return
+
+    exctype, value, traceback = exc_info
+    frames = []
+    while traceback:
+        frames.append(traceback.tb_frame)
+        traceback = traceback.tb_next
+
+    thread = threading.current_thread()
+    frames_by_id = {id(frame): frame for frame in frames}
+    frame = frames[-1]
+    exception = (exctype, value, traceback)
+
+    if hasattr(thread, 'additional_info'):
+        thread.additional_info.pydev_message = 'server exception'
+    try:
+        debugger = pydevd.debugger  # noqa
+    except AttributeError:
+        debugger = pydevd.get_global_debugger()  # noqa
+
+    pydevd_tracing.SetTrace(None)
+
+    debugger.stop_on_unhandled_exception(thread, frame, frames_by_id, exception)

omlish/diag/threads.py

@@ -9,6 +9,90 @@ import traceback
 import typing as ta
 
 
+##
+
+
+def dump_threads(out: ta.IO) -> None:
+    out.write('\n\n')
+
+    thrs_by_tid = {t.ident: t for t in threading.enumerate()}
+
+    for tid, fr in sys._current_frames().items():  # noqa
+        try:
+            thr = thrs_by_tid[tid]
+        except KeyError:
+            thr_rpr = repr(tid)
+        else:
+            thr_rpr = repr(thr)
+
+        tb = traceback.format_stack(fr)
+
+        out.write(f'{thr_rpr}\n')
+        out.write('\n'.join(l.strip() for l in tb))
+        out.write('\n\n')
+
+
+def dump_threads_str() -> str:
+    out = io.StringIO()
+    dump_threads(out)
+    return out.getvalue()
+
+
+##
+
+
+class StoppableThread:
+    def __init__(
+            self,
+            fn: ta.Callable[[], None],
+            interval_s: float,
+            *,
+            tick_immediately: bool = False,
+            start: bool = False,
+            **kwargs: ta.Any,
+    ) -> None:
+        super().__init__()
+        self._fn = fn
+        self._interval_s = interval_s
+        self._tick_immediately = tick_immediately
+        self._thread = threading.Thread(target=self._loop, **kwargs)
+        self._stop_event = threading.Event()
+        if start:
+            self.start()
+
+    @property
+    def thread(self) -> threading.Thread:
+        return self._thread
+
+    @property
+    def ident(self) -> int | None:
+        return self._thread.ident
+
+    def start(self) -> None:
+        return self._thread.start()
+
+    def stop_nowait(self) -> None:
+        self._stop_event.set()
+
+    def stop_wait(self, timeout: float | None = None) -> None:
+        self.stop_nowait()
+        self._thread.join(timeout)
+
+    def _loop(self) -> None:
+        if self._tick_immediately:
+            self._fn()
+
+        while True:
+            self._stop_event.wait(self._interval_s)
+            if self._stop_event.is_set():
+                return
+
+            self._fn()
+
+
+##
+
+
 _DEBUG_THREAD_COUNTER = itertools.count()
 
 
@@ -18,47 +102,23 @@ def create_thread_dump_thread(
         out: ta.TextIO = sys.stderr,
         start: bool = False,
         nodaemon: bool = False,
-) -> threading.Thread:
-    def dump():
-        cthr = threading.current_thread()
-        thrs_by_tid = {t.ident: t for t in threading.enumerate()}
-
-        buf = io.StringIO()
-        for tid, fr in sys._current_frames().items():  # noqa
-            if tid == cthr.ident:
-                continue
-
-            try:
-                thr = thrs_by_tid[tid]
-            except KeyError:
-                thr_rpr = repr(tid)
-            else:
-                thr_rpr = repr(thr)
-
-            tb = traceback.format_stack(fr)
-
-            buf.write(f'{thr_rpr}\n')
-            buf.write('\n'.join(l.strip() for l in tb))
-            buf.write('\n\n')
+) -> StoppableThread:
+    def proc() -> None:
+        try:
+            out.write(dump_threads_str())
+        except Exception as e:  # noqa
+            out.write(repr(e) + '\n\n')
 
-        out.write(buf.getvalue())
-
-    def proc():
-        while True:
-            time.sleep(interval_s)
-            try:
-                dump()
-            except Exception as e:  # noqa
-                out.write(repr(e) + '\n\n')
-
-    dthr = threading.Thread(
-        target=proc,
+    return StoppableThread(
+        proc,
+        interval_s,
         daemon=not nodaemon,
         name=f'thread-dump-thread-{next(_DEBUG_THREAD_COUNTER)}',
+        start=start,
     )
-    if start:
-        dthr.start()
-    return dthr
+
+
+##
 
 
 def create_suicide_thread(
@@ -67,20 +127,43 @@ def create_suicide_thread(
         interval_s: float = 1.,
         parent_thread: threading.Thread | None = None,
         start: bool = False,
-) -> threading.Thread:
+) -> StoppableThread:
+    """Kills process when parent_thread dies."""
+
     if parent_thread is None:
         parent_thread = threading.current_thread()
 
-    def proc():
-        while True:
-            parent_thread.join(interval_s)
-            if not parent_thread.is_alive():
-                os.kill(os.getpid(), sig)
+    def proc() -> None:
+        if not parent_thread.is_alive():
+            os.kill(os.getpid(), sig)
 
-    dthr = threading.Thread(
-        target=proc,
+    return StoppableThread(
+        proc,
+        interval_s,
         name=f'suicide-thread-{next(_DEBUG_THREAD_COUNTER)}',
+        start=start,
+    )
+
+
+##
+
+
+def create_timebomb_thread(
+        delay_s: float,
+        *,
+        sig: int = signal.SIGKILL,
+        interval_s: float = 1.,
+        start: bool = False,
+) -> StoppableThread:
+    def proc() -> None:
+        if time.time() >= deadline:
+            os.kill(os.getpid(), sig)
+
+    deadline = time.time() + delay_s
+
+    return StoppableThread(
+        proc,
+        interval_s,
+        name=f'timebomb-thread-{next(_DEBUG_THREAD_COUNTER)}',
+        start=start,
     )
-    if start:
-        dthr.start()
-    return dthr

omlish/dispatch/_dispatch2.py

@@ -0,0 +1,65 @@
+import abc
+import typing as ta
+import weakref
+
+from .dispatch import find_impl
+
+
+T = ta.TypeVar('T')
+
+
+##
+
+
+class Dispatcher(ta.Generic[T]):
+    def __init__(self) -> None:
+        super().__init__()
+
+        self._impls_by_arg_cls: dict[type, T] = {}
+        self._dispatch_cache: dict[ta.Any, T | None] = {}
+
+        def cache_remove(k, self_ref=weakref.ref(self)):
+            if (ref_self := self_ref()) is not None:
+                cache = ref_self._dispatch_cache  # noqa
+                try:
+                    del cache[k]
+                except KeyError:
+                    pass
+
+        self._cache_remove = cache_remove
+
+        self._cache_token: ta.Any = None
+
+    def cache_size(self) -> int:
+        return len(self._dispatch_cache)
+
+    def register(self, impl: T, cls_col: ta.Iterable[type]) -> T:
+        for cls in cls_col:
+            self._impls_by_arg_cls[cls] = impl
+
+            if self._cache_token is None and hasattr(cls, '__abstractmethods__'):
+                self._cache_token = abc.get_cache_token()
+
+        self._dispatch_cache.clear()
+        return impl
+
+    def dispatch(self, cls: type) -> T | None:
+        if self._cache_token is not None and (current_token := abc.get_cache_token()) != self._cache_token:
+            self._dispatch_cache.clear()
+            self._cache_token = current_token
+
+        cls_ref = weakref.ref(cls)
+        try:
+            return self._dispatch_cache[cls_ref]
+        except KeyError:
+            pass
+        del cls_ref
+
+        impl: T | None
+        try:
+            impl = self._impls_by_arg_cls[cls]
+        except KeyError:
+            impl = find_impl(cls, self._impls_by_arg_cls)
+
+        self._dispatch_cache[weakref.ref(cls, self._cache_remove)] = impl
+        return impl

omlish/dispatch/_dispatch3.py

@@ -0,0 +1,104 @@
+import abc
+import typing as ta
+import weakref
+
+from .dispatch import find_impl
+
+
+T = ta.TypeVar('T')
+
+
+##
+
+
+class DispatchCacheProtocol(ta.Protocol[T]):
+    def size(self) -> int: ...
+    def prepare(self, cls: type) -> None: ...
+    def clear(self) -> None: ...
+    def put(self, cls: type, impl: T) -> None: ...
+    def get(self, cls: type) -> T: ...  # Raises[KeyError]
+
+
+class DispatcherProtocol(ta.Protocol[T]):
+    def cache_size(self) -> int: ...
+    def register(self, impl: T, cls_col: ta.Iterable[type]) -> T: ...
+    def dispatch(self, cls: type) -> T | None: ...
+
+
+##
+
+
+class DispatchCache(DispatchCacheProtocol[T]):
+    def __init__(self) -> None:
+        super().__init__()
+
+        self._dct: dict[ta.Any, T] = {}
+
+        def remove(k, self_ref=weakref.ref(self)):
+            if (ref_self := self_ref()) is not None:
+                dct = ref_self._dct  # noqa
+                try:
+                    del dct[k]
+                except KeyError:
+                    pass
+
+        self._remove = remove
+
+        self._token: ta.Any = None
+
+    def size(self) -> int:
+        return len(self._dct)
+
+    def prepare(self, cls: type) -> None:
+        if self._token is None and hasattr(cls, '__abstractmethods__'):
+            self._token = abc.get_cache_token()
+
+        self.clear()
+
+    def clear(self) -> None:
+        if self._dct:
+            self._dct.clear()
+
+    def put(self, cls: type, impl: T) -> None:
+        self._dct[weakref.ref(cls, self._remove)] = impl
+
+    def get(self, cls: type) -> T:
+        if self._token is not None and (current_token := abc.get_cache_token()) != self._token:
+            self._dct.clear()
+            self._token = current_token
+
+        cls_ref = weakref.ref(cls)
+        return self._dct[cls_ref]
+
+
+class Dispatcher(DispatcherProtocol[T]):
+    def __init__(self) -> None:
+        super().__init__()
+
+        self._impls_by_arg_cls: dict[type, T] = {}
+        self._cache: DispatchCache[T | None] = DispatchCache()
+
+    def cache_size(self) -> int:
+        return self._cache.size()
+
+    def register(self, impl: T, cls_col: ta.Iterable[type]) -> T:
+        for cls in cls_col:
+            self._impls_by_arg_cls[cls] = impl
+            self._cache.prepare(cls)
+
+        return impl
+
+    def dispatch(self, cls: type) -> T | None:
+        try:
+            return self._cache.get(cls)
+        except KeyError:
+            pass
+
+        impl: T | None
+        try:
+            impl = self._impls_by_arg_cls[cls]
+        except KeyError:
+            impl = find_impl(cls, self._impls_by_arg_cls)
+
+        self._cache.put(cls, impl)
+        return impl

omlish/docker.py

@@ -18,13 +18,14 @@ import datetime
 import re
 import shlex
 import subprocess
+import sys
 import typing as ta
 
 from . import check
 from . import dataclasses as dc
-from . import json
 from . import lang
 from . import marshal as msh
+from .formats import json
 
 
 if ta.TYPE_CHECKING:
@@ -168,3 +169,17 @@ def timebomb_payload(delay_s: float, name: str = 'omlish-docker-timebomb') -> st
         'sh -c \'killall5 -9 -o $PPID -o $$ ; kill 1\''
         ') &'
     )
+
+
+##
+
+
+_LIKELY_IN_DOCKER_PATTERN = re.compile(r'^overlay / .*/docker/')
+
+
+def is_likely_in_docker() -> bool:
+    if sys.platform != 'linux':
+        return False
+    with open('/proc/mounts') as f:  # type: ignore
+        ls = f.readlines()
+    return any(_LIKELY_IN_DOCKER_PATTERN.match(l) for l in ls)

omlish/fnpairs.py

@@ -4,11 +4,7 @@ TODO:
   - csv
   - csvloader
   - cbor
-  - cloudpickle
   - alt json backends
- - compression
-  - snappy
-  - lz4
  - wrapped (wait for usecase)
 """
 import abc
@@ -29,6 +25,7 @@ if ta.TYPE_CHECKING:
     import tomllib as _tomllib
 
     import cloudpickle as _cloudpickle
+    import json5 as _json5
     import lz4.frame as _lz4_frame
     import snappy as _snappy
     import yaml as _yaml
@@ -44,6 +41,7 @@ else:
     _tomllib = lang.proxy_import('tomllib')
 
     _cloudpickle = lang.proxy_import('cloudpickle')
+    _json5 = lang.proxy_import('json5')
     _lz4_frame = lang.proxy_import('lz4.frame')
     _snappy = lang.proxy_import('snappy')
     _yaml = lang.proxy_import('yaml')
@@ -391,6 +389,15 @@ class Cloudpickle(ObjectBytes_):
         return _cloudpickle.loads(t)
 
 
+@_register_extension('json5')
+class Json5(ObjectStr_):
+    def forward(self, f: ta.Any) -> str:
+        return _json5.dumps(f)
+
+    def backward(self, t: str) -> ta.Any:
+        return _json5.loads(t)
+
+
 @_register_extension('yml', 'yaml')
 class Yaml(ObjectStr_):
     def forward(self, f: ta.Any) -> str:

omlish/{configs → formats}/dotenv.py

@@ -20,6 +20,7 @@
 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# https://github.com/theskumar/python-dotenv/tree/4d505f2c9bc3569791e64bca0f2e4300f43df0e0/src/dotenv
 import abc
 import codecs
 import contextlib
@@ -332,6 +333,7 @@ class DotEnv:
         encoding: str | None = None,
         interpolate: bool = True,
         override: bool = True,
+        env: ta.Mapping[str, str] | None = None,
     ) -> None:
         super().__init__()
         self.path: StrPath | None = path
@@ -341,6 +343,7 @@ class DotEnv:
         self.encoding: str | None = encoding
         self.interpolate: bool = interpolate
         self.override: bool = override
+        self.env = env or {}
 
     @contextlib.contextmanager
     def _get_stream(self) -> ta.Iterator[ta.IO[str]]:
@@ -364,7 +367,7 @@ class DotEnv:
         raw_values = self.parse()
 
         if self.interpolate:
-            self._dict = resolve_variables(raw_values, override=self.override)
+            self._dict = resolve_variables(raw_values, override=self.override, env=self.env)
         else:
             self._dict = dict(raw_values)
 
@@ -376,22 +379,7 @@ class DotEnv:
                 if mapping.key is not None:
                     yield mapping.key, mapping.value
 
-    def set_as_environment_variables(self) -> bool:
-        """Load the current dotenv as system environment variable."""
-        if not self.dict():
-            return False
-
-        for k, v in self.dict().items():
-            if k in os.environ and not self.override:
-                continue
-            if v is not None:
-                os.environ[k] = v
-
-        return True
-
     def get(self, key: str) -> str | None:
-        """
-        """
         data = self.dict()
 
         if key in data:
@@ -525,8 +513,9 @@ def unset_key(
 
 
 def resolve_variables(
-    values: ta.Iterable[tuple[str, str | None]],
-    override: bool,
+        values: ta.Iterable[tuple[str, str | None]],
+        override: bool,
+        env: ta.Mapping[str, str],
 ) -> dict[str, str | None]:
     new_values: dict[str, str | None] = {}
 
@@ -535,14 +524,14 @@ def resolve_variables(
             result = None
         else:
             atoms = parse_variables(value)
-            env: dict[str, str | None] = {}
+            aenv: dict[str, str | None] = {}
             if override:
-                env.update(os.environ)
-                env.update(new_values)
+                aenv.update(env)
+                aenv.update(new_values)
             else:
-                env.update(new_values)
-                env.update(os.environ)
-            result = ''.join(atom.resolve(env) for atom in atoms)
+                aenv.update(new_values)
+                aenv.update(env)
+            result = ''.join(atom.resolve(aenv) for atom in atoms)
 
         new_values[name] = result
 
@@ -556,6 +545,7 @@ def dotenv_values(
     verbose: bool = False,
     interpolate: bool = True,
     encoding: str | None = 'utf-8',
+    env: ta.Mapping[str, str] | None = None,
 ) -> dict[str, str | None]:
     """
     Parse a .env file and return its content as a dict.
@@ -583,4 +573,5 @@ def dotenv_values(
         interpolate=interpolate,
         override=True,
         encoding=encoding,
+        env=env,
     ).dict()

omlish/{json.py → formats/json.py}

@@ -125,7 +125,7 @@ import functools
 import json as _json
 import typing as ta
 
-from . import lang
+from .. import lang
 
 
 if ta.TYPE_CHECKING:
@@ -160,6 +160,7 @@ PRETTY_KWARGS: ta.Mapping[str, ta.Any] = dict(
 dump_pretty: ta.Callable[..., bytes] = functools.partial(dump, **PRETTY_KWARGS)  # type: ignore
 dumps_pretty: ta.Callable[..., str] = functools.partial(dumps, **PRETTY_KWARGS)
 
+
 ##