omlish 0.0.0.dev224__py3-none-any.whl → 0.0.0.dev225__py3-none-any.whl

omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev224'
-__revision__ = '0980d15b9ba0f06f68cf7bdb68fbc8626696075f'
+__version__ = '0.0.0.dev225'
+__revision__ = '99eb1740a6c8647c20c80278b6d0bedff0f7c103'
 
 
 #

omlish/asyncs/asyncio/subprocesses.py

@@ -10,9 +10,9 @@ import sys
 import typing as ta
 
 from ...lite.check import check
-from ...subprocesses import AbstractAsyncSubprocesses
-from ...subprocesses import SubprocessRun
-from ...subprocesses import SubprocessRunOutput
+from ...subprocesses.async_ import AbstractAsyncSubprocesses
+from ...subprocesses.run import SubprocessRun
+from ...subprocesses.run import SubprocessRunOutput
 from .timeouts import asyncio_maybe_timeout
 
 
@@ -157,19 +157,19 @@ class AsyncioSubprocesses(AbstractAsyncSubprocesses):
             timeout: ta.Optional[float] = None,
             **kwargs: ta.Any,
     ) -> ta.AsyncGenerator[asyncio.subprocess.Process, None]:
-        fac: ta.Any
-        if shell:
-            fac = functools.partial(
-                asyncio.create_subprocess_shell,
-                check.single(cmd),
-            )
-        else:
-            fac = functools.partial(
-                asyncio.create_subprocess_exec,
-                *cmd,
-            )
-
         with self.prepare_and_wrap( *cmd, shell=shell, **kwargs) as (cmd, kwargs):  # noqa
+            fac: ta.Any
+            if shell:
+                fac = functools.partial(
+                    asyncio.create_subprocess_shell,
+                    check.single(cmd),
+                )
+            else:
+                fac = functools.partial(
+                    asyncio.create_subprocess_exec,
+                    *cmd,
+                )
+
             proc: asyncio.subprocess.Process = await fac(**kwargs)
             try:
                 yield proc

omlish/asyncs/asyncs.py

@@ -55,7 +55,6 @@ async def async_list(fn: ta.Callable[..., ta.AsyncIterator[T]], *args, **kwargs)
 
 
 class SyncableIterable(ta.Generic[T]):
-
     def __init__(self, obj) -> None:
         super().__init__()
         self._obj = obj

omlish/bootstrap/sys.py

@@ -23,13 +23,13 @@ if ta.TYPE_CHECKING:
     from .. import libc
     from ..formats import dotenv
     from ..logs import all as logs
-    from ..os import pidfile
+    from ..os.pidfiles import pidfile
 
 else:
     libc = lang.proxy_import('..libc', __package__)
     logs = lang.proxy_import('..logs', __package__)
     dotenv = lang.proxy_import('..formats.dotenv', __package__)
-    pidfile = lang.proxy_import('..os.pidfile', __package__)
+    pidfile = lang.proxy_import('..os.pidfiles.pidfile', __package__)
 
 
 ##

omlish/dataclasses/impl/metaclass.py

@@ -188,8 +188,13 @@ class Data(
     metaclass=DataMeta,
 ):
     def __init__(self, *args, **kwargs):
+        # Typechecking barrier
         super().__init__(*args, **kwargs)
 
+    def __init_subclass__(cls, **kwargs):
+        # Typechecking barrier
+        super().__init_subclass__(**kwargs)
+
     def __post_init__(self, *args, **kwargs) -> None:
         try:
             spi = super().__post_init__  # type: ignore  # noqa

omlish/http/coro/server.py

@@ -65,8 +65,6 @@ import typing as ta
 
 from ...lite.check import check
 from ...sockets.addresses import SocketAddress
-from ...sockets.handlers import SocketHandler_
-from ...sockets.io import SocketIoPair
 from ..handlers import HttpHandler
 from ..handlers import HttpHandlerRequest
 from ..handlers import HttpHandlerResponseData
@@ -423,6 +421,9 @@ class CoroHttpServer:
     def coro_handle(self) -> ta.Generator[Io, ta.Optional[bytes], None]:
         return self._coro_run_handler(self._coro_handle_one())
 
+    class Close(Exception):  # noqa
+        pass
+
     def _coro_run_handler(
             self,
             gen: ta.Generator[
@@ -462,7 +463,7 @@ class CoroHttpServer:
 
                 try:
                     o = gen.send(i)
-                except EOFError:
+                except self.Close:
                     return
                 except StopIteration:
                     break
@@ -491,7 +492,7 @@ class CoroHttpServer:
                 break
 
         if isinstance(parsed, EmptyParsedHttpResult):
-            raise EOFError  # noqa
+            raise self.Close
 
         if isinstance(parsed, ParseHttpRequestError):
             err = self._build_error(
@@ -581,53 +582,3 @@ class CoroHttpServer:
             handler_response.close()
 
             raise
-
-
-##
-
-
-class CoroHttpServerSocketHandler(SocketHandler_):
-    def __init__(
-            self,
-            server_factory: CoroHttpServerFactory,
-            *,
-            log_handler: ta.Optional[ta.Callable[[CoroHttpServer, CoroHttpServer.AnyLogIo], None]] = None,
-    ) -> None:
-        super().__init__()
-
-        self._server_factory = server_factory
-        self._log_handler = log_handler
-
-    def __call__(self, client_address: SocketAddress, fp: SocketIoPair) -> None:
-        server = self._server_factory(client_address)
-
-        gen = server.coro_handle()
-
-        o = next(gen)
-        while True:
-            if isinstance(o, CoroHttpServer.AnyLogIo):
-                i = None
-                if self._log_handler is not None:
-                    self._log_handler(server, o)
-
-            elif isinstance(o, CoroHttpServer.ReadIo):
-                i = fp.r.read(o.sz)
-
-            elif isinstance(o, CoroHttpServer.ReadLineIo):
-                i = fp.r.readline(o.sz)
-
-            elif isinstance(o, CoroHttpServer.WriteIo):
-                i = None
-                fp.w.write(o.data)
-                fp.w.flush()
-
-            else:
-                raise TypeError(o)
-
-            try:
-                if i is not None:
-                    o = gen.send(i)
-                else:
-                    o = next(gen)
-            except StopIteration:
-                break

omlish/http/coro/simple.py

@@ -21,7 +21,7 @@ from ..parsing import HttpRequestParser
 from ..versions import HttpProtocolVersion
 from ..versions import HttpProtocolVersions
 from .server import CoroHttpServer
-from .server import CoroHttpServerSocketHandler
+from .sockets import CoroHttpServerSocketHandler
 
 
 if ta.TYPE_CHECKING:

omlish/http/coro/sockets.py

@@ -0,0 +1,59 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import typing as ta
+
+from ...sockets.addresses import SocketAddress
+from ...sockets.handlers import SocketHandler_
+from ...sockets.io import SocketIoPair
+from .server import CoroHttpServer
+from .server import CoroHttpServerFactory
+
+
+##
+
+
+class CoroHttpServerSocketHandler(SocketHandler_):
+    def __init__(
+            self,
+            server_factory: CoroHttpServerFactory,
+            *,
+            log_handler: ta.Optional[ta.Callable[[CoroHttpServer, CoroHttpServer.AnyLogIo], None]] = None,
+    ) -> None:
+        super().__init__()
+
+        self._server_factory = server_factory
+        self._log_handler = log_handler
+
+    def __call__(self, client_address: SocketAddress, fp: SocketIoPair) -> None:
+        server = self._server_factory(client_address)
+
+        gen = server.coro_handle()
+
+        o = next(gen)
+        while True:
+            if isinstance(o, CoroHttpServer.AnyLogIo):
+                i = None
+                if self._log_handler is not None:
+                    self._log_handler(server, o)
+
+            elif isinstance(o, CoroHttpServer.ReadIo):
+                i = fp.r.read(o.sz)
+
+            elif isinstance(o, CoroHttpServer.ReadLineIo):
+                i = fp.r.readline(o.sz)
+
+            elif isinstance(o, CoroHttpServer.WriteIo):
+                i = None
+                fp.w.write(o.data)
+                fp.w.flush()
+
+            else:
+                raise TypeError(o)
+
+            try:
+                if i is not None:
+                    o = gen.send(i)
+                else:
+                    o = next(gen)
+            except StopIteration:
+                break

omlish/lang/__init__.py

@@ -138,6 +138,7 @@ from .generators import (  # noqa
 from .imports import (  # noqa
     can_import,
     import_all,
+    import_attr,
     import_module,
     import_module_attr,
     lazy_import,

omlish/lang/imports.py

@@ -108,6 +108,28 @@ def import_module_attr(dotted_path: str) -> ta.Any:
         raise AttributeError(f'Module {module_name!r} has no attr {class_name!r}') from None
 
 
+def import_attr(dotted_path: str) -> ta.Any:
+    parts = dotted_path.split('.')
+    mod: ta.Any = None
+    mod_pos = 0
+    while mod_pos < len(parts):
+        mod_name = '.'.join(parts[:mod_pos + 1])
+        try:
+            mod = importlib.import_module(mod_name)
+        except ImportError:
+            break
+        mod_pos += 1
+    if mod is None:
+        raise ImportError(dotted_path)
+    obj = mod
+    for att_pos in range(mod_pos, len(parts)):
+        obj = getattr(obj, parts[att_pos])
+    return obj
+
+
+##
+
+
 SPECIAL_IMPORTABLE: ta.AbstractSet[str] = frozenset([
     '__init__.py',
     '__main__.py',

omlish/libc.py

@@ -392,6 +392,14 @@ if LINUX or DARWIN:
         return CMSG_ALIGN(ct.sizeof(cmsghdr)) + sz
 
     def sendfd(sock, fd, data='.'):
+        """
+        Note: stdlib as of 3.7:
+
+          https://github.com/python/cpython/blob/84ed9a68bd9a13252b376b21a9167dabae254325/Lib/multiprocessing/reduction.py#L141
+
+        But still kept around due to other use of cmsg machinery.
+        """  # noqa
+
         if not data:
             raise ValueError(data)
 
@@ -424,6 +432,8 @@ if LINUX or DARWIN:
         return libc.sendmsg(sock, msgh, 0)
 
     def recvfd(sock, buf_len=4096):
+        """See sendfd."""
+
         if buf_len < 1:
             raise ValueError(buf_len)
 

omlish/multiprocessing/__init__.py

@@ -4,13 +4,6 @@ import typing as _ta
 from .. import lang as _lang
 
 
-from .death import (  # noqa
-    BaseDeathpact,
-    Deathpact,
-    NopDeathpact,
-    PipeDeathpact,
-)
-
 from .proxies import (  # noqa
     DummyValueProxy,
     ValueProxy,

omlish/os/pidfiles/manager.py

@@ -0,0 +1,97 @@
+# ruff: noqa: UP007
+# @omlish-lite
+import contextlib
+import os
+import threading
+import typing as ta
+import weakref
+
+from ...lite.check import check
+from .pidfile import Pidfile
+
+
+##
+
+
+class _PidfileManager:
+    def __new__(cls, *args, **kwargs):  # noqa
+        raise TypeError
+
+    _lock: ta.ClassVar[threading.Lock] = threading.Lock()
+    _installed: ta.ClassVar[bool] = False
+    _pidfile_threads: ta.ClassVar[ta.MutableMapping[Pidfile, threading.Thread]] = weakref.WeakKeyDictionary()
+
+    @classmethod
+    def _before_fork(cls) -> None:
+        cls._lock.acquire()
+
+    @classmethod
+    def _after_fork_in_parent(cls) -> None:
+        cls._lock.release()
+
+    @classmethod
+    def _after_fork_in_child(cls) -> None:
+        th = threading.current_thread()
+        for pf, pf_th in list(cls._pidfile_threads.items()):
+            if pf_th is not th:
+                pf.close()
+                del cls._pidfile_threads[pf]
+
+        cls._lock.release()
+
+    #
+
+    @classmethod
+    def _install(cls) -> None:
+        check.state(not cls._installed)
+
+        os.register_at_fork(
+            before=cls._before_fork,
+            after_in_parent=cls._after_fork_in_parent,
+            after_in_child=cls._after_fork_in_child,
+        )
+
+        cls._installed = True
+
+    @classmethod
+    def install(cls) -> bool:
+        with cls._lock:
+            if cls._installed:
+                return False
+
+            cls._install()
+            return True
+
+    @classmethod
+    @contextlib.contextmanager
+    def inheritable_pidfile_context(
+            cls,
+            path: str,
+            *,
+            inheritable: bool = True,
+            **kwargs: ta.Any,
+    ) -> ta.Iterator[Pidfile]:
+        check.arg(inheritable)
+
+        cls.install()
+
+        pf = Pidfile(
+            path,
+            inheritable=False,
+            **kwargs,
+        )
+
+        with cls._lock:
+            cls._pidfile_threads[pf] = threading.current_thread()
+        try:
+            with pf:
+                os.set_inheritable(check.not_none(pf.fileno()), True)
+                yield pf
+
+        finally:
+            with cls._lock:
+                del cls._pidfile_threads[pf]
+
+
+def open_inheritable_pidfile(path: str, **kwargs: ta.Any) -> ta.ContextManager[Pidfile]:
+    return _PidfileManager.inheritable_pidfile_context(path, **kwargs)  # noqa

omlish/os/pidfiles/pidfile.py

@@ -0,0 +1,142 @@
+# ruff: noqa: UP007
+# @omlish-lite
+"""
+TODO:
+ - reliable pid retrieval
+  - contents are *ignored*, just advisory
+  - check double-check:
+   - 1) get pid of flock holder
+   - 2) get pidfd to that
+   - 3) recheck current pid of flock holder == that pid
+  - racy as to if it's a different actual process as initial check, just with same pid, but due to 'identity' / semantic
+    meaning of the named pidfile the processes are considered equivalent
+"""
+import fcntl
+import os
+import signal
+import typing as ta
+
+
+##
+
+
+class Pidfile:
+    def __init__(
+            self,
+            path: str,
+            *,
+            inheritable: bool = True,
+    ) -> None:
+        super().__init__()
+
+        self._path = path
+        self._inheritable = inheritable
+
+    def __repr__(self) -> str:
+        return f'{self.__class__.__name__}({self._path!r})'
+
+    #
+
+    _f: ta.TextIO
+
+    def fileno(self) -> ta.Optional[int]:
+        if hasattr(self, '_f'):
+            return self._f.fileno()
+        else:
+            return None
+
+    #
+
+    def __enter__(self) -> 'Pidfile':
+        fd = os.open(self._path, os.O_RDWR | os.O_CREAT, 0o600)
+
+        try:
+            if self._inheritable:
+                os.set_inheritable(fd, True)
+
+            f = os.fdopen(fd, 'r+')
+
+        except Exception:
+            try:
+                os.close(fd)
+            except Exception:  # noqa
+                pass
+            raise
+
+        self._f = f
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
+    #
+
+    def __getstate__(self):
+        state = self.__dict__.copy()
+
+        if '_f' in state:
+            if os.get_inheritable(fd := state.pop('_f').fileno()):
+                state['__fd'] = fd
+
+        return state
+
+    def __setstate__(self, state):
+        if '_f' in state:
+            raise RuntimeError
+
+        if '__fd' in state:
+            state['_f'] = os.fdopen(state.pop('__fd'), 'r+')
+
+        self.__dict__.update(state)
+
+    #
+
+    def close(self) -> bool:
+        if not hasattr(self, '_f'):
+            return False
+
+        self._f.close()
+        del self._f
+        return True
+
+    def try_lock(self) -> bool:
+        try:
+            fcntl.flock(self._f, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            return True
+
+        except OSError:
+            return False
+
+    def ensure_locked(self) -> None:
+        if not self.try_lock():
+            raise RuntimeError('Could not get lock')
+
+    #
+
+    def write(self, pid: ta.Optional[int] = None) -> None:
+        self.ensure_locked()
+
+        if pid is None:
+            pid = os.getpid()
+
+        self._f.seek(0)
+        self._f.truncate()
+        self._f.write(f'{pid}\n')
+        self._f.flush()
+
+    def clear(self) -> None:
+        self.ensure_locked()
+
+        self._f.seek(0)
+        self._f.truncate()
+
+    def read(self) -> int:
+        if self.try_lock():
+            raise RuntimeError('Got lock')
+
+        self._f.seek(0)
+        return int(self._f.read())  # FIXME: could be empty or hold old value, race w proc start
+
+    def kill(self, sig: int = signal.SIGTERM) -> None:
+        pid = self.read()
+        os.kill(pid, sig)  # FIXME: Still racy - pidfd_send_signal?

omlish/secrets/crypto.py

@@ -76,7 +76,6 @@ class Crypto(abc.ABC):
 
 
 class FernetCrypto(Crypto):
-
     def generate_key(self) -> bytes:
         return cry_fernet.Fernet.generate_key()
 
@@ -98,7 +97,7 @@ class FernetCrypto(Crypto):
             raise DecryptionError from e
 
 
-class AesgsmCrypto(Crypto):
+class AesgcmCrypto(Crypto):
     """https://stackoverflow.com/a/59835994"""
 
     def generate_key(self) -> bytes:

omlish/secrets/openssl.py

@@ -33,7 +33,7 @@ else:
 DEFAULT_KEY_SIZE = 64
 
 
-def generate_key(self, sz: int = DEFAULT_KEY_SIZE) -> bytes:
+def generate_key(sz: int = DEFAULT_KEY_SIZE) -> bytes:
     # !! https://docs.openssl.org/3.0/man7/passphrase-encoding/
     # Must not contain null bytes!
     return secrets.token_hex(sz).encode('ascii')

omlish/secrets/tempssl.py

@@ -6,9 +6,9 @@ import tempfile
 import typing as ta
 
 from ..lite.cached import cached_nullary
-from ..subprocesses import SubprocessRun
-from ..subprocesses import SubprocessRunOutput
-from ..subprocesses import subprocesses
+from ..subprocesses.run import SubprocessRun
+from ..subprocesses.run import SubprocessRunnable
+from ..subprocesses.run import SubprocessRunOutput
 from .ssl import SslCert
 
 
@@ -18,7 +18,7 @@ class TempSslCert(ta.NamedTuple):
 
 
 @dc.dataclass(frozen=True)
-class TempSslCertGenerator:
+class TempSslCertGenerator(SubprocessRunnable[TempSslCert]):
     @cached_nullary
     def temp_dir(self) -> str:
         return tempfile.mkdtemp()
@@ -64,9 +64,6 @@ class TempSslCertGenerator:
             temp_dir=self.temp_dir(),
         )
 
-    def run(self) -> TempSslCert:
-        return self.handle_run_output(subprocesses.run_(self.make_run()))
-
 
 def generate_temp_localhost_ssl_cert() -> TempSslCert:
     return TempSslCertGenerator().run()

omlish/sockets/handlers.py

@@ -17,3 +17,7 @@ class SocketHandler_(abc.ABC):  # noqa
     @abc.abstractmethod
     def __call__(self, addr: SocketAddress, f: SocketIoPair) -> None:
         raise NotImplementedError
+
+
+class SocketHandlerClose(Exception):  # noqa
+    pass

omlish/sockets/server/handlers.py

@@ -3,6 +3,7 @@
 import abc
 import concurrent.futures as cf
 import dataclasses as dc
+import logging
 import socket
 import typing as ta
 
@@ -132,3 +133,24 @@ class ExecutorSocketServerHandler(SocketServerHandler_):
 
     def __call__(self, conn: SocketAndAddress) -> None:
         self.executor.submit(self.handler, conn)
+
+
+#
+
+
+@dc.dataclass(frozen=True)
+class ExceptionLoggingSocketServerHandler(SocketServerHandler_):
+    handler: SocketServerHandler
+    log: logging.Logger
+
+    ignored: ta.Optional[ta.Container[ta.Type[Exception]]] = None
+
+    def __call__(self, conn: SocketAndAddress) -> None:
+        try:
+            return self.handler(conn)
+
+        except Exception as e:  # noqa
+            if (ignored := self.ignored) is None or type(e) not in ignored:
+                self.log.exception('Error in handler %r for conn %r', self.handler, conn)
+
+            raise