omlish 0.0.0.dev129__tar.gz → 0.0.0.dev130__tar.gz

{omlish-0.0.0.dev129/omlish.egg-info → omlish-0.0.0.dev130}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omlish
-Version: 0.0.0.dev129
+Version: 0.0.0.dev130
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause

{omlish-0.0.0.dev129 → omlish-0.0.0.dev130}/omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev129'
-__revision__ = '997d6cc6f7e56919cf06a3d284f3d64e3383fc81'
+__version__ = '0.0.0.dev130'
+__revision__ = '04fd1bb83aa7700c60be8dc97b6530ddf15b6dd7'
 
 
 #

{omlish-0.0.0.dev129 → omlish-0.0.0.dev130}/omlish/lite/check.py

@@ -3,6 +3,7 @@ import typing as ta
 
 
 T = ta.TypeVar('T')
+SizedT = ta.TypeVar('SizedT', bound=ta.Sized)
 
 
 def check_isinstance(v: ta.Any, spec: ta.Union[ta.Type[T], tuple]) -> T:
@@ -84,3 +85,15 @@ def check_not_in(v: T, c: ta.Container[T]) -> T:
 def check_single(vs: ta.Iterable[T]) -> T:
     [v] = vs
     return v
+
+
+def check_empty(v: SizedT) -> SizedT:
+    if len(v):
+        raise ValueError(v)
+    return v
+
+
+def check_non_empty(v: SizedT) -> SizedT:
+    if not len(v):
+        raise ValueError(v)
+    return v

omlish-0.0.0.dev130/omlish/lite/fdio/corohttp.py

@@ -0,0 +1,135 @@
+import socket
+import typing as ta
+
+from ..check import check_isinstance
+from ..check import check_none
+from ..check import check_not_none
+from ..check import check_state
+from ..http.coroserver import CoroHttpServer
+from ..http.handlers import HttpHandler
+from ..io import IncrementalWriteBuffer
+from ..io import ReadableListBuffer
+from ..socket import SocketAddress
+from .handlers import SocketFdIoHandler
+
+
+class CoroHttpServerConnectionFdIoHandler(SocketFdIoHandler):
+    def __init__(
+            self,
+            addr: SocketAddress,
+            sock: socket.socket,
+            handler: HttpHandler,
+            *,
+            read_size: int = 0x10000,
+            write_size: int = 0x10000,
+    ) -> None:
+        check_state(not sock.getblocking())
+
+        super().__init__(addr, sock)
+
+        self._handler = handler
+        self._read_size = read_size
+        self._write_size = write_size
+
+        self._read_buf = ReadableListBuffer()
+        self._write_buf: IncrementalWriteBuffer | None = None
+
+        self._coro_srv = CoroHttpServer(
+            addr,
+            handler=self._handler,
+        )
+        self._srv_coro: ta.Optional[ta.Generator[CoroHttpServer.Io, ta.Optional[bytes], None]] = self._coro_srv.coro_handle()  # noqa
+
+        self._cur_io: CoroHttpServer.Io | None = None
+        self._next_io()
+
+    #
+
+    def _next_io(self) -> None:  # noqa
+        coro = check_not_none(self._srv_coro)
+
+        d: bytes | None = None
+        o = self._cur_io
+        while True:
+            if o is None:
+                try:
+                    if d is not None:
+                        o = coro.send(d)
+                        d = None
+                    else:
+                        o = next(coro)
+                except StopIteration:
+                    self.close()
+                    o = None
+                    break
+
+            if isinstance(o, CoroHttpServer.AnyLogIo):
+                print(o)
+                o = None
+
+            elif isinstance(o, CoroHttpServer.ReadIo):
+                if (d := self._read_buf.read(o.sz)) is None:
+                    break
+                o = None
+
+            elif isinstance(o, CoroHttpServer.ReadLineIo):
+                if (d := self._read_buf.read_until(b'\n')) is None:
+                    break
+                o = None
+
+            elif isinstance(o, CoroHttpServer.WriteIo):
+                check_none(self._write_buf)
+                self._write_buf = IncrementalWriteBuffer(o.data, write_size=self._write_size)
+                break
+
+            else:
+                raise TypeError(o)
+
+        self._cur_io = o
+
+    #
+
+    def readable(self) -> bool:
+        return True
+
+    def writable(self) -> bool:
+        return self._write_buf is not None
+
+    #
+
+    def on_readable(self) -> None:
+        try:
+            buf = check_not_none(self._sock).recv(self._read_size)
+        except BlockingIOError:
+            return
+        except ConnectionResetError:
+            self.close()
+            return
+        if not buf:
+            self.close()
+            return
+
+        self._read_buf.feed(buf)
+
+        if isinstance(self._cur_io, CoroHttpServer.AnyReadIo):
+            self._next_io()
+
+    def on_writable(self) -> None:
+        check_isinstance(self._cur_io, CoroHttpServer.WriteIo)
+        wb = check_not_none(self._write_buf)
+        while wb.rem > 0:
+            def send(d: bytes) -> int:
+                try:
+                    return check_not_none(self._sock).send(d)
+                except ConnectionResetError:
+                    self.close()
+                    return 0
+                except BlockingIOError:
+                    return 0
+            if not wb.write(send):
+                break
+
+        if wb.rem < 1:
+            self._write_buf = None
+            self._cur_io = None
+            self._next_io()

omlish-0.0.0.dev130/omlish/lite/fdio/handlers.py

@@ -0,0 +1,67 @@
+# ruff: noqa: UP006 UP007
+import abc
+import socket
+import typing as ta
+
+from ..check import check_not_none
+from ..socket import SocketAddress
+
+
+class FdIoHandler(abc.ABC):
+    @abc.abstractmethod
+    def fd(self) -> int:
+        raise NotImplementedError
+
+    #
+
+    @property
+    @abc.abstractmethod
+    def closed(self) -> bool:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def close(self) -> None:
+        raise NotImplementedError
+
+    #
+
+    def readable(self) -> bool:
+        return False
+
+    def writable(self) -> bool:
+        return False
+
+    #
+
+    def on_readable(self) -> None:
+        raise TypeError
+
+    def on_writable(self) -> None:
+        raise TypeError
+
+    def on_error(self, exc: ta.Optional[BaseException] = None) -> None:  # noqa
+        pass
+
+
+class SocketFdIoHandler(FdIoHandler, abc.ABC):
+    def __init__(
+            self,
+            addr: SocketAddress,
+            sock: socket.socket,
+    ) -> None:
+        super().__init__()
+
+        self._addr = addr
+        self._sock: ta.Optional[socket.socket] = sock
+
+    def fd(self) -> int:
+        return check_not_none(self._sock).fileno()
+
+    @property
+    def closed(self) -> bool:
+        return self._sock is None
+
+    def close(self) -> None:
+        if self._sock is not None:
+            self._sock.close()
+        self._sock = None

omlish-0.0.0.dev130/omlish/lite/fdio/kqueue.py

@@ -0,0 +1,102 @@
+# ruff: noqa: UP006 UP007
+import errno
+import select
+import sys
+import typing as ta
+
+from .pollers import FdIoPoller
+
+
+KqueueFdIoPoller: ta.Optional[ta.Type[FdIoPoller]]
+if sys.platform == 'darwin' or sys.platform.startswith('freebsd'):
+
+    class _KqueueFdIoPoller(FdIoPoller):
+        DEFAULT_MAX_EVENTS = 1000
+
+        def __init__(
+                self,
+                *,
+                max_events: int = DEFAULT_MAX_EVENTS,
+        ) -> None:
+            super().__init__()
+
+            self._max_events = max_events
+
+            self._kqueue: ta.Optional[ta.Any] = None
+
+        #
+
+        def _get_kqueue(self) -> 'select.kqueue':
+            if (kq := self._kqueue) is not None:
+                return kq
+            kq = select.kqueue()
+            self._kqueue = kq
+            return kq
+
+        def close(self) -> None:
+            if self._kqueue is not None:
+                self._kqueue.close()
+                self._kqueue = None
+
+        def reopen(self) -> None:
+            for fd in self._readable:
+                self._register_readable(fd)
+            for fd in self._writable:
+                self._register_writable(fd)
+
+        #
+
+        def _register_readable(self, fd: int) -> None:
+            self._control(fd, select.KQ_FILTER_READ, select.KQ_EV_ADD)
+
+        def _register_writable(self, fd: int) -> None:
+            self._control(fd, select.KQ_FILTER_WRITE, select.KQ_EV_ADD)
+
+        def _unregister_readable(self, fd: int) -> None:
+            self._control(fd, select.KQ_FILTER_READ, select.KQ_EV_DELETE)
+
+        def _unregister_writable(self, fd: int) -> None:
+            self._control(fd, select.KQ_FILTER_WRITE, select.KQ_EV_DELETE)
+
+        def _control(self, fd: int, filter: int, flags: int) -> None:  # noqa
+            ke = select.kevent(fd, filter=filter, flags=flags)
+            kq = self._get_kqueue()
+            try:
+                kq.control([ke], 0)
+
+            except OSError as exc:
+                if exc.errno == errno.EBADF:
+                    # log.debug('EBADF encountered in kqueue. Invalid file descriptor %s', ke.ident)
+                    pass
+                elif exc.errno == errno.ENOENT:
+                    # Can happen when trying to remove an already closed socket
+                    pass
+                else:
+                    raise
+
+        #
+
+        def poll(self, timeout: ta.Optional[float]) -> FdIoPoller.PollResult:
+            kq = self._get_kqueue()
+            try:
+                kes = kq.control(None, self._max_events, timeout)
+
+            except OSError as exc:
+                if exc.errno == errno.EINTR:
+                    return FdIoPoller.PollResult(msg='EINTR encountered in poll', exc=exc)
+                else:
+                    raise
+
+            r: ta.List[int] = []
+            w: ta.List[int] = []
+            for ke in kes:
+                if ke.filter == select.KQ_FILTER_READ:
+                    r.append(ke.ident)
+                if ke.filter == select.KQ_FILTER_WRITE:
+                    w.append(ke.ident)
+
+            return FdIoPoller.PollResult(r, w)
+
+    KqueueFdIoPoller = _KqueueFdIoPoller
+else:
+    KqueueFdIoPoller = None

omlish-0.0.0.dev130/omlish/lite/fdio/manager.py

@@ -0,0 +1,48 @@
+# ruff: noqa: UP006 UP007
+import typing as ta
+
+from .handlers import FdIoHandler
+from .pollers import FdIoPoller
+
+
+class FdIoManager:
+    def __init__(
+            self,
+            poller: FdIoPoller,
+    ) -> None:
+        super().__init__()
+
+        self._poller = poller
+
+        self._handlers: ta.Dict[int, FdIoHandler] = {}  # Preserves insertion order
+
+    def register(self, h: FdIoHandler) -> None:
+        if (hid := id(h)) in self._handlers:
+            raise KeyError(h)
+        self._handlers[hid] = h
+
+    def unregister(self, h: FdIoHandler) -> None:
+        del self._handlers[id(h)]
+
+    def poll(self, *, timeout: float = 1.) -> None:
+        hs = list(self._handlers.values())
+        rd = {h.fd(): h for h in hs if h.readable()}
+        wd = {h.fd(): h for h in hs if h.writable()}
+
+        self._poller.update(set(rd), set(wd))
+
+        pr = self._poller.poll(timeout)
+
+        for f in pr.r:
+            if not (h := rd[f]).closed:
+                h.on_readable()
+        for f in pr.w:
+            if not (h := wd[f]).closed:
+                h.on_writable()
+
+        hs = list(self._handlers.values())
+        nh = {}
+        for h in hs:
+            if not h.closed:
+                nh[id(h)] = h
+        self._handlers = nh

omlish-0.0.0.dev130/omlish/lite/fdio/pollers.py

@@ -0,0 +1,212 @@
+# ruff: noqa: UP006 UP007
+import abc
+import dataclasses as dc
+import errno
+import select
+import typing as ta
+
+
+##
+
+
+class FdIoPoller(abc.ABC):
+    def __init__(self) -> None:
+        super().__init__()
+
+        self._readable: ta.Set[int] = set()
+        self._writable: ta.Set[int] = set()
+
+    #
+
+    def close(self) -> None:  # noqa
+        pass
+
+    def reopen(self) -> None:  # noqa
+        pass
+
+    #
+
+    @property
+    @ta.final
+    def readable(self) -> ta.AbstractSet[int]:
+        return self._readable
+
+    @property
+    @ta.final
+    def writable(self) -> ta.AbstractSet[int]:
+        return self._writable
+
+    #
+
+    @ta.final
+    def register_readable(self, fd: int) -> bool:
+        if fd in self._readable:
+            return False
+        self._readable.add(fd)
+        self._register_readable(fd)
+        return True
+
+    @ta.final
+    def register_writable(self, fd: int) -> bool:
+        if fd in self._writable:
+            return False
+        self._writable.add(fd)
+        self._register_writable(fd)
+        return True
+
+    @ta.final
+    def unregister_readable(self, fd: int) -> bool:
+        if fd not in self._readable:
+            return False
+        self._readable.discard(fd)
+        self._unregister_readable(fd)
+        return True
+
+    @ta.final
+    def unregister_writable(self, fd: int) -> bool:
+        if fd not in self._writable:
+            return False
+        self._writable.discard(fd)
+        self._unregister_writable(fd)
+        return True
+
+    #
+
+    def _register_readable(self, fd: int) -> None:  # noqa
+        pass
+
+    def _register_writable(self, fd: int) -> None:  # noqa
+        pass
+
+    def _unregister_readable(self, fd: int) -> None:  # noqa
+        pass
+
+    def _unregister_writable(self, fd: int) -> None:  # noqa
+        pass
+
+    #
+
+    def update(
+            self,
+            r: ta.AbstractSet[int],
+            w: ta.AbstractSet[int],
+    ) -> None:
+        for f in r - self._readable:
+            self.register_readable(f)
+        for f in w - self._writable:
+            self.register_writable(f)
+        for f in self._readable - r:
+            self.unregister_readable(f)
+        for f in self._writable - w:
+            self.unregister_writable(f)
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class PollResult:
+        r: ta.Sequence[int] = ()
+        w: ta.Sequence[int] = ()
+
+        inv: ta.Sequence[int] = ()
+
+        msg: ta.Optional[str] = None
+        exc: ta.Optional[BaseException] = None
+
+    @abc.abstractmethod
+    def poll(self, timeout: ta.Optional[float]) -> PollResult:
+        raise NotImplementedError
+
+
+##
+
+
+class SelectFdIoPoller(FdIoPoller):
+    def poll(self, timeout: ta.Optional[float]) -> FdIoPoller.PollResult:
+        try:
+            r, w, x = select.select(
+                self._readable,
+                self._writable,
+                [],
+                timeout,
+            )
+
+        except OSError as exc:
+            if exc.errno == errno.EINTR:
+                return FdIoPoller.PollResult(msg='EINTR encountered in poll', exc=exc)
+            elif exc.errno == errno.EBADF:
+                return FdIoPoller.PollResult(msg='EBADF encountered in poll', exc=exc)
+            else:
+                raise
+
+        return FdIoPoller.PollResult(r, w)
+
+
+##
+
+
+PollFdIoPoller: ta.Optional[ta.Type[FdIoPoller]]
+if hasattr(select, 'poll'):
+
+    class _PollFdIoPoller(FdIoPoller):
+        def __init__(self) -> None:
+            super().__init__()
+
+            self._poller = select.poll()
+
+        #
+
+        _READ = select.POLLIN | select.POLLPRI | select.POLLHUP
+        _WRITE = select.POLLOUT
+
+        def _register_readable(self, fd: int) -> None:
+            self._update_registration(fd)
+
+        def _register_writable(self, fd: int) -> None:
+            self._update_registration(fd)
+
+        def _unregister_readable(self, fd: int) -> None:
+            self._update_registration(fd)
+
+        def _unregister_writable(self, fd: int) -> None:
+            self._update_registration(fd)
+
+        def _update_registration(self, fd: int) -> None:
+            r = fd in self._readable
+            w = fd in self._writable
+            if r or w:
+                self._poller.register(fd, (self._READ if r else 0) | (self._WRITE if w else 0))
+            else:
+                self._poller.unregister(fd)
+
+        #
+
+        def poll(self, timeout: ta.Optional[float]) -> FdIoPoller.PollResult:
+            polled: ta.List[ta.Tuple[int, int]]
+            try:
+                polled = self._poller.poll(timeout * 1000 if timeout is not None else None)
+
+            except OSError as exc:
+                if exc.errno == errno.EINTR:
+                    return FdIoPoller.PollResult(msg='EINTR encountered in poll', exc=exc)
+                else:
+                    raise
+
+            r: ta.List[int] = []
+            w: ta.List[int] = []
+            inv: ta.List[int] = []
+            for fd, mask in polled:
+                if mask & select.POLLNVAL:
+                    self._poller.unregister(fd)
+                    self._readable.discard(fd)
+                    self._writable.discard(fd)
+                    inv.append(fd)
+                    continue
+                if mask & self._READ:
+                    r.append(fd)
+                if mask & self._WRITE:
+                    w.append(fd)
+            return FdIoPoller.PollResult(r, w, inv=inv)
+
+    PollFdIoPoller = _PollFdIoPoller
+else:
+    PollFdIoPoller = None