omlish 0.0.0.dev102__py3-none-any.whl → 0.0.0.dev104__py3-none-any.whl

omlish/io/trampoline.py

@@ -0,0 +1,293 @@
+"""
+TODO:
+ - https://docs.python.org/3/library/zlib.html#zlib.compressobj
+"""
+import abc
+import contextlib
+import io
+import typing as ta
+
+from .. import check
+from .. import lang
+from ..concurrent import threadlets as tls
+from ..sync import ConditionDeque
+
+
+if ta.TYPE_CHECKING:
+    import threading
+
+    from . import pyio  # noqa
+
+else:
+    threading = lang.proxy_import('threading')
+
+    pyio = lang.proxy_import('.pyio', __package__)
+
+
+T = ta.TypeVar('T')
+
+BytesLike: ta.TypeAlias = ta.Any
+
+BufferedReader = io.BufferedReader
+# BufferedReader = pyio.BufferedReader
+
+
+##
+
+
+class ProxyReadFile:
+    def __init__(self, read: ta.Callable[[int], bytes | BaseException]) -> None:
+        super().__init__()
+
+        self._read = read
+        self._eof = False
+        self._closed = False
+
+    #
+
+    def read(self, n: int, /) -> bytes:
+        if self._closed:
+            raise RuntimeError('Closed')
+        if self._eof:
+            return b''
+        d = self._read(n)
+        if isinstance(d, BaseException):
+            raise d
+        if not d:
+            self._eof = True
+        return d
+
+    def readall(self, *args, **kwargs):
+        raise TypeError  # FIXME
+
+    def readinto(self, b: BytesLike) -> int | None:
+        d = self.read(len(b))
+        if d:
+            b[:len(d)] = d
+        return len(d)
+
+    #
+
+    def readable(self) -> bool:
+        return not (self._eof or self._closed)
+
+    @property
+    def closed(self) -> bool:
+        return self._closed
+
+    def close(self) -> None:
+        self._closed = True
+
+    #
+
+    def flush(self) -> None:
+        pass
+
+    def seekable(self) -> bool:
+        return False
+
+    def seek(self, n: int, /) -> ta.Any:
+        raise TypeError
+
+
+##
+
+
+class NeedMore(lang.Marker):
+    pass
+
+
+class Exited(lang.Marker):
+    pass
+
+
+class Shutdown(BaseException):  # noqa
+    pass
+
+
+IoTrampolineTarget: ta.TypeAlias = ta.Callable[[io.BufferedReader], ta.ContextManager[ta.Callable[[], bytes]]]
+
+
+class IoTrampoline(lang.Abstract):
+    def __init__(
+            self,
+            target: IoTrampolineTarget,
+            *,
+            buffer_size: int | None = None,
+    ) -> None:
+        super().__init__()
+
+        self._target = target
+        self._buffer_size = buffer_size
+
+    def _make_buffered_reader(self, raw: ta.Any) -> io.BufferedReader:
+        return io.BufferedReader(
+            raw,
+            **(dict(buffer_size=self._buffer_size) if self._buffer_size is not None else {}),
+        )
+
+    @abc.abstractmethod
+    def close(self, timeout: float | None = None) -> None:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def __enter__(self) -> ta.Self:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def feed(self, *data: bytes) -> ta.Iterable[bytes]:
+        raise NotImplementedError
+
+
+#
+
+
+class ThreadIoTrampoline(IoTrampoline):
+    def __init__(self, target: IoTrampolineTarget, **kwargs: ta.Any) -> None:
+        super().__init__(target, **kwargs)
+
+        self._in: ConditionDeque[bytes | BaseException] = ConditionDeque()
+        self._out: ConditionDeque[
+            bytes |  # noqa
+            type[NeedMore] |
+            type[Exited] |
+            BaseException
+        ] = ConditionDeque()
+
+        self._proxy = ProxyReadFile(self._read)
+
+        self._thread = threading.Thread(target=self._thread_proc, daemon=True)
+
+    #
+
+    def close(self, timeout: float | None = None) -> None:
+        if not self._thread.is_alive():
+            return
+        self._out.push(Shutdown())
+        self._thread.join(timeout)
+        if self._thread.is_alive():
+            if timeout is not None:
+                raise TimeoutError
+            else:
+                raise RuntimeError('Failed to join thread')
+
+    #
+
+    def __enter__(self) -> ta.Self:
+        self._thread.start()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
+    #
+
+    def _read(self, n: int, /) -> bytes | BaseException:
+        return self._in.pop(if_empty=lambda: self._out.push(NeedMore))
+
+    def _thread_proc(self) -> None:
+        try:
+            with contextlib.closing(self._make_buffered_reader(self._proxy)) as bf:  # noqa
+                with self._target(bf) as read:
+                    while out := read():
+                        self._out.push(out)
+                    self._out.push(out)
+        except BaseException as e:
+            self._out.push(e)
+            raise
+        finally:
+            self._out.push(Exited)
+
+    def feed(self, *data: bytes) -> ta.Iterable[bytes]:
+        self._in.push(*data)
+        while True:
+            e = self._out.pop()
+            if e is NeedMore:
+                break
+            elif isinstance(e, BaseException):
+                raise e
+            elif e is Exited:
+                raise RuntimeError('IO thread exited')
+            elif isinstance(e, bytes):
+                yield e
+                if not e:
+                    return
+            else:
+                raise TypeError(e)
+
+
+#
+
+
+class ThreadletIoTrampoline(IoTrampoline):
+    def __init__(
+            self,
+            target: IoTrampolineTarget,
+            threadlets: tls.Threadlets = tls.GREENLET_THREADLETS,
+            ** kwargs: ta.Any,
+    ) -> None:
+        super().__init__(target, **kwargs)
+
+        self._proxy = ProxyReadFile(self._read)
+        self._tl: tls.Threadlet = threadlets.spawn(self._g_proc)
+
+    #
+
+    def close(self, timeout: float | None = None) -> None:
+        if self._tl.dead:
+            return
+        out = self._tl.switch(Shutdown())
+        if out is not Exited or not self._tl.dead:
+            raise RuntimeError
+
+    #
+
+    def __enter__(self) -> ta.Self:
+        out = self._tl.switch()
+        if out is not NeedMore:
+            raise RuntimeError
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
+    #
+
+    def _read(self, n: int, /) -> bytes:
+        out = check.not_none(self._tl.parent).switch(NeedMore)
+        return out
+
+    def _g_proc(self) -> ta.Any:
+        try:
+            with contextlib.closing(self._make_buffered_reader(self._proxy)) as bf:  # noqa
+                with self._target(bf) as read:
+                    while out := read():
+                        e = check.not_none(self._tl.parent).switch(out)
+                        if e is not NeedMore:
+                            raise TypeError(e)  # noqa
+                    e = check.not_none(self._tl.parent).switch(out)
+                    if not isinstance(e, Shutdown):
+                        raise TypeError(e)  # noqa
+            return Exited
+        except BaseException as e:
+            check.not_none(self._tl.parent).throw(e)
+            raise
+
+    def feed(self, *data: bytes) -> ta.Iterable[bytes]:
+        i: bytes | type[NeedMore]
+        for i in data:
+            while True:
+                e = self._tl.switch(i)
+                i = NeedMore
+                if e is NeedMore:
+                    break
+                elif isinstance(e, bytes):
+                    yield e
+                    if not e:
+                        return
+                else:
+                    raise TypeError(e)

omlish/lite/cached.py

@@ -1,7 +1,11 @@
 import functools
+import typing as ta
 
 
-class cached_nullary:  # noqa
+T = ta.TypeVar('T')
+
+
+class _cached_nullary:  # noqa
     def __init__(self, fn):
         super().__init__()
         self._fn = fn
@@ -16,3 +20,7 @@ class cached_nullary:  # noqa
     def __get__(self, instance, owner):  # noqa
         bound = instance.__dict__[self._fn.__name__] = self.__class__(self._fn.__get__(instance, owner))
         return bound
+
+
+def cached_nullary(fn: ta.Callable[..., T]) -> ta.Callable[..., T]:
+    return _cached_nullary(fn)

omlish/lite/contextmanagers.py

@@ -1,4 +1,38 @@
+# ruff: noqa: UP007
 import contextlib
+import typing as ta
+
+from .check import check_not_none
+from .check import check_state
+
+
+T = ta.TypeVar('T')
+ExitStackedT = ta.TypeVar('ExitStackedT', bound='ExitStacked')
+
+
+##
+
+
+class ExitStacked:
+    _exit_stack: ta.Optional[contextlib.ExitStack] = None
+
+    def __enter__(self: ExitStackedT) -> ExitStackedT:
+        check_state(self._exit_stack is None)
+        es = self._exit_stack = contextlib.ExitStack()
+        es.__enter__()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if (es := self._exit_stack) is None:
+            return None
+        return es.__exit__(exc_type, exc_val, exc_tb)
+
+    def _enter_context(self, cm: ta.ContextManager[T]) -> T:
+        es = check_not_none(self._exit_stack)
+        return es.enter_context(cm)
+
+
+##
 
 
 @contextlib.contextmanager

omlish/lite/marshal.py

@@ -12,6 +12,8 @@ import datetime
 import decimal
 import enum
 import fractions
+import functools
+import threading
 import typing as ta
 import uuid
 import weakref  # noqa
@@ -150,7 +152,7 @@ class DataclassObjMarshaler(ObjMarshaler):
         return {k: m.marshal(getattr(o, k)) for k, m in self.fs.items()}
 
     def unmarshal(self, o: ta.Any) -> ta.Any:
-        return self.ty(**{k: self.fs[k].unmarshal(v) for k, v in o.items() if self.nonstrict or k in self.fs})
+        return self.ty(**{k: self.fs[k].unmarshal(v) for k, v in o.items() if not self.nonstrict or k in self.fs})
 
 
 @dc.dataclass(frozen=True)
@@ -210,7 +212,10 @@ class UuidObjMarshaler(ObjMarshaler):
         return uuid.UUID(o)
 
 
-_OBJ_MARSHALERS: ta.Dict[ta.Any, ObjMarshaler] = {
+##
+
+
+_DEFAULT_OBJ_MARSHALERS: ta.Dict[ta.Any, ObjMarshaler] = {
     **{t: NopObjMarshaler() for t in (type(None),)},
     **{t: CastObjMarshaler(t) for t in (int, float, str, bool)},
     **{t: Base64ObjMarshaler(t) for t in (bytes, bytearray)},
@@ -239,20 +244,19 @@ _OBJ_MARSHALER_GENERIC_ITERABLE_TYPES: ta.Dict[ta.Any, type] = {
 }
 
 
-def register_opj_marshaler(ty: ta.Any, m: ObjMarshaler) -> None:
-    if ty in _OBJ_MARSHALERS:
-        raise KeyError(ty)
-    _OBJ_MARSHALERS[ty] = m
-
-
-def _make_obj_marshaler(ty: ta.Any) -> ObjMarshaler:
+def _make_obj_marshaler(
+        ty: ta.Any,
+        rec: ta.Callable[[ta.Any], ObjMarshaler],
+        *,
+        nonstrict_dataclasses: bool = False,
+) -> ObjMarshaler:
     if isinstance(ty, type):
         if abc.ABC in ty.__bases__:
             impls = [  # type: ignore
                 PolymorphicObjMarshaler.Impl(
                     ity,
                     ity.__qualname__,
-                    get_obj_marshaler(ity),
+                    rec(ity),
                 )
                 for ity in deep_subclasses(ty)
                 if abc.ABC not in ity.__bases__
@@ -268,7 +272,8 @@ def _make_obj_marshaler(ty: ta.Any) -> ObjMarshaler:
         if dc.is_dataclass(ty):
             return DataclassObjMarshaler(
                 ty,
-                {f.name: get_obj_marshaler(f.type) for f in dc.fields(ty)},
+                {f.name: rec(f.type) for f in dc.fields(ty)},
+                nonstrict=nonstrict_dataclasses,
             )
 
     if is_generic_alias(ty):
@@ -278,7 +283,7 @@ def _make_obj_marshaler(ty: ta.Any) -> ObjMarshaler:
             pass
         else:
             k, v = ta.get_args(ty)
-            return MappingObjMarshaler(mt, get_obj_marshaler(k), get_obj_marshaler(v))
+            return MappingObjMarshaler(mt, rec(k), rec(v))
 
         try:
             st = _OBJ_MARSHALER_GENERIC_ITERABLE_TYPES[ta.get_origin(ty)]
@@ -286,33 +291,71 @@ def _make_obj_marshaler(ty: ta.Any) -> ObjMarshaler:
             pass
         else:
             [e] = ta.get_args(ty)
-            return IterableObjMarshaler(st, get_obj_marshaler(e))
+            return IterableObjMarshaler(st, rec(e))
 
         if is_union_alias(ty):
-            return OptionalObjMarshaler(get_obj_marshaler(get_optional_alias_arg(ty)))
+            return OptionalObjMarshaler(rec(get_optional_alias_arg(ty)))
 
     raise TypeError(ty)
 
 
-def get_obj_marshaler(ty: ta.Any) -> ObjMarshaler:
-    try:
-        return _OBJ_MARSHALERS[ty]
-    except KeyError:
-        pass
-
-    p = ProxyObjMarshaler()
-    _OBJ_MARSHALERS[ty] = p
-    try:
-        m = _make_obj_marshaler(ty)
-    except Exception:
-        del _OBJ_MARSHALERS[ty]
-        raise
-    else:
-        p.m = m
+##
+
+
+_OBJ_MARSHALERS_LOCK = threading.RLock()
+
+_OBJ_MARSHALERS: ta.Dict[ta.Any, ObjMarshaler] = dict(_DEFAULT_OBJ_MARSHALERS)
+
+_OBJ_MARSHALER_PROXIES: ta.Dict[ta.Any, ProxyObjMarshaler] = {}
+
+
+def register_opj_marshaler(ty: ta.Any, m: ObjMarshaler) -> None:
+    with _OBJ_MARSHALERS_LOCK:
+        if ty in _OBJ_MARSHALERS:
+            raise KeyError(ty)
         _OBJ_MARSHALERS[ty] = m
+
+
+def get_obj_marshaler(
+        ty: ta.Any,
+        *,
+        no_cache: bool = False,
+        **kwargs: ta.Any,
+) -> ObjMarshaler:
+    with _OBJ_MARSHALERS_LOCK:
+        if not no_cache:
+            try:
+                return _OBJ_MARSHALERS[ty]
+            except KeyError:
+                pass
+
+        try:
+            return _OBJ_MARSHALER_PROXIES[ty]
+        except KeyError:
+            pass
+
+        rec = functools.partial(
+            get_obj_marshaler,
+            no_cache=no_cache,
+            **kwargs,
+        )
+
+        p = ProxyObjMarshaler()
+        _OBJ_MARSHALER_PROXIES[ty] = p
+        try:
+            m = _make_obj_marshaler(ty, rec, **kwargs)
+        finally:
+            del _OBJ_MARSHALER_PROXIES[ty]
+        p.m = m
+
+        if not no_cache:
+            _OBJ_MARSHALERS[ty] = m
         return m
 
 
+##
+
+
 def marshal_obj(o: ta.Any, ty: ta.Any = None) -> ta.Any:
     return get_obj_marshaler(ty if ty is not None else type(o)).marshal(o)
 

omlish/lite/pidfile.py

@@ -30,7 +30,7 @@ class Pidfile:
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
-        if self._f is not None:
+        if hasattr(self, '_f'):
             self._f.close()
             del self._f
 

omlish/lite/subprocesses.py

@@ -110,3 +110,21 @@ def subprocess_try_output(
 def subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:
     out = subprocess_try_output(*args, **kwargs)
     return out.decode().strip() if out is not None else None
+
+
+##
+
+
+def subprocess_close(
+        proc: subprocess.Popen,
+        timeout: ta.Optional[float] = None,
+) -> None:
+    # TODO: terminate, sleep, kill
+    if proc.stdout:
+        proc.stdout.close()
+    if proc.stderr:
+        proc.stderr.close()
+    if proc.stdin:
+        proc.stdin.close()
+
+    proc.wait(timeout)

omlish/specs/__init__.py

@@ -4,6 +4,8 @@ TODO:
  - iceberg
  - jsonnet
  - jsonpatch
+ - jsonpath
  - jsonpointer.py
  - protobuf
+ - irc? or in net?
 """

omlish/sync.py

@@ -3,6 +3,7 @@ TODO:
  - sync (lol) w/ asyncs.anyio
  - atomics
 """
+import collections
 import threading
 import typing as ta
 
@@ -68,3 +69,57 @@ class LazyFn(ta.Generic[T]):
             self._v = lang.just(self._fn())
         self._once.do(do)
         return self._v.must()
+
+
+class ConditionDeque(ta.Generic[T]):
+    def __init__(
+            self,
+            *,
+            cond: ta.Optional['threading.Condition'] = None,
+            deque: collections.deque[T] | None = None,
+
+            lock: ta.Optional['threading.RLock'] = None,
+            maxlen: int | None = None,
+            init: ta.Iterable[T] | None = None,
+    ) -> None:
+        super().__init__()
+
+        if cond is None:
+            cond = threading.Condition(lock=lock)
+        if deque is None:
+            deque = collections.deque(maxlen=maxlen)
+        if init:
+            deque.extend(init)
+
+        self._cond = cond
+        self._deque = deque
+
+    @property
+    def cond(self) -> 'threading.Condition':
+        return self._cond
+
+    @property
+    def deque(self) -> collections.deque[T]:
+        return self._deque
+
+    def push(
+            self,
+            *items: T,
+            n: int = 1,
+    ) -> None:
+        with self.cond:
+            self.deque.extend(items)
+            self.cond.notify(n)
+
+    def pop(
+            self,
+            timeout: float | None = None,
+            *,
+            if_empty: ta.Callable[[], None] | None = None,
+    ) -> T:
+        with self.cond:
+            if not self.deque and if_empty is not None:
+                if_empty()
+            while not self.deque:
+                self.cond.wait(timeout)
+            return self.deque.popleft()

{omlish-0.0.0.dev102.dist-info → omlish-0.0.0.dev104.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omlish
-Version: 0.0.0.dev102
+Version: 0.0.0.dev104
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause