omlish 0.0.0.dev226__tar.gz → 0.0.0.dev227__tar.gz

{omlish-0.0.0.dev226/omlish.egg-info → omlish-0.0.0.dev227}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: omlish
-Version: 0.0.0.dev226
+Version: 0.0.0.dev227
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause

{omlish-0.0.0.dev226 → omlish-0.0.0.dev227}/omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev226'
-__revision__ = '1f3419dac98e9db37df48a9a952f780d17bf31ed'
+__version__ = '0.0.0.dev227'
+__revision__ = '2faa3e7f9120edd3e6dd79fad6401fd87d89f5cf'
 
 
 #

{omlish-0.0.0.dev226 → omlish-0.0.0.dev227}/omlish/diag/lslocks.py

@@ -19,7 +19,7 @@ from ..subprocesses.run import SubprocessRunOutput
 
 
 @dc.dataclass(frozen=True)
-class LsLocksItem:
+class LslocksItem:
     """https://manpages.ubuntu.com/manpages/lunar/man8/lslocks.8.html"""
 
     command: str
@@ -38,7 +38,7 @@ class LsLocksItem:
 
 
 @dc.dataclass(frozen=True)
-class LsLocksCommand(SubprocessRunnable):
+class LslocksCommand(SubprocessRunnable[ta.List[LslocksItem]]):
     pid: ta.Optional[int] = None
     no_inaccessible: bool = False
 
@@ -56,9 +56,9 @@ class LsLocksCommand(SubprocessRunnable):
             stderr='devnull',
         )
 
-    def handle_run_output(self, output: SubprocessRunOutput) -> ta.List[LsLocksItem]:
+    def handle_run_output(self, output: SubprocessRunOutput) -> ta.List[LslocksItem]:
         buf = check.not_none(output.stdout).decode().strip()
         if not buf:
             return []
         obj = json.loads(buf)
-        return unmarshal_obj(obj['locks'], ta.List[LsLocksItem])
+        return unmarshal_obj(obj['locks'], ta.List[LslocksItem])

{omlish-0.0.0.dev226 → omlish-0.0.0.dev227}/omlish/diag/lsof.py

@@ -151,7 +151,6 @@ class LsofItem:
 
     @classmethod
     def from_prefixes(cls, dct: ta.Mapping[str, ta.Any]) -> 'LsofItem':
-        print(dct)
         kw: ta.Dict[str, ta.Any] = {
             fld.name: val
             for pfx, val in dct.items()
@@ -216,7 +215,8 @@ LsofItem._DEFAULT_PREFIXES = ''.join(LsofItem._FIELDS_BY_PREFIX)  # noqa
 
 
 @dc.dataclass(frozen=True)
-class LsofCommand(SubprocessRunnable[ta.List[LsofItem]]):
+class \
+        LsofCommand(SubprocessRunnable[ta.List[LsofItem]]):
     pid: ta.Optional[int] = None
     file: ta.Optional[str] = None
 

{omlish-0.0.0.dev226 → omlish-0.0.0.dev227}/omlish/os/pidfiles/manager.py

@@ -15,7 +15,10 @@ from .pidfile import Pidfile
 
 class _PidfileManager:
     """
-    Manager for controlled inheritance of Pidfiles across forks.
+    Manager for controlled inheritance of Pidfiles across forks in the presence of multiple threads. There is of course
+    no safe or correct way to mix the use of fork and multiple active threads, and one should never write code which
+    does so, but in the Real World one may still find oneself in such a situation outside of their control (such as when
+    running under Pycharm's debugger which forces the use of forked multiprocessing).
 
     Not implemented as an instantiated class as there is no way to unregister at_fork listeners, and because Pidfiles
     may be pickled and there must be no possibility of accidentally unpickling and instantiating a new instance of the

{omlish-0.0.0.dev226 → omlish-0.0.0.dev227}/omlish/os/pidfiles/pidfile.py

@@ -109,7 +109,7 @@ class Pidfile:
         del self._f
         return True
 
-    def try_lock(self) -> bool:
+    def try_acquire_lock(self) -> bool:
         try:
             fcntl.flock(self._f, fcntl.LOCK_EX | fcntl.LOCK_NB)
             return True
@@ -117,14 +117,29 @@ class Pidfile:
         except OSError:
             return False
 
-    def ensure_locked(self) -> None:
-        if not self.try_lock():
-            raise RuntimeError('Could not get lock')
+    #
+
+    class Error(Exception):
+        pass
+
+    class LockedError(Error):
+        pass
+
+    def acquire_lock(self) -> None:
+        if not self.try_acquire_lock():
+            raise self.LockedError
+
+    class NotLockedError(Error):
+        pass
+
+    def ensure_cannot_lock(self) -> None:
+        if self.try_acquire_lock():
+            raise self.NotLockedError
 
     #
 
     def write(self, pid: ta.Optional[int] = None) -> None:
-        self.ensure_locked()
+        self.acquire_lock()
 
         if pid is None:
             pid = os.getpid()
@@ -135,18 +150,23 @@ class Pidfile:
         self._f.flush()
 
     def clear(self) -> None:
-        self.ensure_locked()
+        self.acquire_lock()
 
         self._f.seek(0)
         self._f.truncate()
 
-    def read(self) -> int:
-        if self.try_lock():
-            raise RuntimeError('Got lock')
+    #
+
+    def read(self) -> ta.Optional[int]:
+        self.ensure_cannot_lock()
 
         self._f.seek(0)
-        return int(self._f.read())
+        buf = self._f.read()
+        if not buf:
+            return None
+        return int(buf)
 
     def kill(self, sig: int = signal.SIGTERM) -> None:
-        pid = self.read()
+        if (pid := self.read()) is None:
+            raise self.Error(f'Pidfile locked but empty')
         os.kill(pid, sig)

omlish-0.0.0.dev227/omlish/os/pidfiles/pinning.py

@@ -0,0 +1,246 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+"""
+Strategies:
+ - linux
+  - get pid of owner (lslocks or F_GETLK)
+  - open pidfd to owner pid
+  - re-check pid of owner
+ - darwin
+  - get pids of referrers (lsof)
+  - read pid from file
+  - ensure pid is in referrers
+  - optionally loop
+  - ? setup pid death watcher? still a race
+"""
+import abc
+import contextlib
+import os.path
+import shutil
+import sys
+import time
+import typing as ta
+
+from ...diag.lslocks import LslocksCommand
+from ...diag.lsof import LsofCommand
+from ...lite.timeouts import Timeout
+from ...lite.timeouts import TimeoutLike
+from ...subprocesses.sync import subprocesses  # noqa
+from .pidfile import Pidfile
+
+
+##
+
+
+class PidfilePinner(abc.ABC):
+    def __init__(
+            self,
+            *,
+            sleep_s: float = .1,
+    ) -> None:
+        super().__init__()
+
+        self._sleep_s = sleep_s
+
+    @classmethod
+    @abc.abstractmethod
+    def is_available(cls) -> bool:
+        raise NotImplementedError
+
+    class NoOwnerError(Exception):
+        pass
+
+    @abc.abstractmethod
+    def _pin_pidfile_owner(self, pidfile: Pidfile, timeout: Timeout) -> ta.ContextManager[int]:
+        raise NotImplementedError
+
+    @contextlib.contextmanager
+    def pin_pidfile_owner(
+            self,
+            path: str,
+            *,
+            timeout: ta.Optional[TimeoutLike] = None,
+    ) -> ta.Iterator[int]:
+        timeout = Timeout.of(timeout)
+
+        if not os.path.isfile(path):
+            raise self.NoOwnerError
+
+        with Pidfile(path, inheritable=False) as pf:
+            try:
+                with self._pin_pidfile_owner(pf, timeout) as pid:
+                    yield pid
+
+            except Pidfile.NotLockedError:
+                raise self.NoOwnerError from None
+
+    @classmethod
+    def default_impl(cls) -> ta.Type['PidfilePinner']:
+        for impl in [
+            LslocksPidfdPidfilePinner,
+            LsofPidfilePinner,
+        ]:
+            if impl.is_available():
+                return impl
+        return UnverifiedPidfilePinner
+
+
+##
+
+
+class UnverifiedPidfilePinner(PidfilePinner):
+    @classmethod
+    def is_available(cls) -> bool:
+        return True
+
+    @contextlib.contextmanager
+    def _pin_pidfile_owner(self, pidfile: Pidfile, timeout: Timeout) -> ta.Iterator[int]:
+        while (pid := pidfile.read()) is None:
+            time.sleep(self._sleep_s)
+            timeout()
+
+        yield pid
+
+
+##
+
+
+class LsofPidfilePinner(PidfilePinner):
+    """
+    Fundamentally wrong, but still better than nothing. Simply reads the file contents and ensures a valid contained pid
+    has the file open via `lsof`.
+    """
+
+    @classmethod
+    def is_available(cls) -> bool:
+        return shutil.which('lsof') is not None
+
+    def _try_read_and_verify(self, pf: Pidfile, timeout: Timeout) -> ta.Optional[int]:
+        if (initial_pid := pf.read()) is None:
+            return None
+
+        lsof_output = LsofCommand(
+            # pid=initial_pid,
+            file=os.path.abspath(pf.path),
+        ).run(
+            timeout=timeout,
+        )
+
+        lsof_pids: ta.Set[int] = set()
+        for li in lsof_output:
+            if li.pid is None:
+                continue
+            try:
+                li_pid = int(li.pid)
+            except ValueError:
+                continue
+            lsof_pids.add(li_pid)
+
+        if initial_pid not in lsof_pids:
+            return None
+
+        if (reread_pid := pf.read()) is None or reread_pid != initial_pid:
+            return None
+
+        return reread_pid
+
+    @contextlib.contextmanager
+    def _pin_pidfile_owner(self, pidfile: Pidfile, timeout: Timeout) -> ta.Iterator[int]:
+        while (pid := self._try_read_and_verify(pidfile, timeout)) is None:
+            time.sleep(self._sleep_s)
+            timeout()
+
+        yield pid
+
+
+##
+
+
+class LslocksPidfdPidfilePinner(PidfilePinner):
+    """
+    Finds the locking pid via `lslocks`, opens a pidfd, then re-runs `lslocks` and rechecks the locking pid is the same.
+    """
+
+    @classmethod
+    def is_available(cls) -> bool:
+        return sys.platform == 'linux' and shutil.which('lslocks') is not None
+
+    def _read_locking_pid(self, path: str, timeout: Timeout) -> int:
+        lsl_output = LslocksCommand().run(timeout=timeout)
+
+        lsl_pids = {
+            li.pid
+            for li in lsl_output
+            if li.path == path
+            and li.type == 'FLOCK'
+        }
+        if not lsl_pids:
+            raise self.NoOwnerError
+        if len(lsl_pids) != 1:
+            raise RuntimeError(f'Multiple locks on file: {path}')
+
+        [pid] = lsl_pids
+        return pid
+
+    class _Result(ta.NamedTuple):
+        pid: int
+        pidfd: int
+
+    def _try_read_and_verify(
+            self,
+            pidfile: Pidfile,
+            timeout: Timeout,
+    ) -> ta.Optional[_Result]:
+        path = os.path.abspath(pidfile.path)
+        initial_pid = self._read_locking_pid(path, timeout)
+
+        try:
+            pidfd = os.open(f'/proc/{initial_pid}', os.O_RDONLY)
+        except FileNotFoundError:
+            raise self.NoOwnerError from None
+
+        try:
+            reread_pid = self._read_locking_pid(path, timeout)
+            if reread_pid != initial_pid:
+                os.close(pidfd)
+                return None
+
+            return self._Result(initial_pid, pidfd)
+
+        except BaseException:
+            os.close(pidfd)
+            raise
+
+    @contextlib.contextmanager
+    def _pin_pidfile_owner(
+            self,
+            pidfile: Pidfile,
+            timeout: Timeout,
+    ) -> ta.Iterator[int]:
+        while (res := self._try_read_and_verify(pidfile, timeout)) is None:
+            time.sleep(self._sleep_s)
+            timeout()
+
+        try:
+            yield res.pid
+        finally:
+            os.close(res.pidfd)
+
+
+##
+
+
+if __name__ == '__main__':
+    def _main() -> None:
+        argparse = __import__('argparse')
+        parser = argparse.ArgumentParser()
+        parser.add_argument('file')
+        args = parser.parse_args()
+
+        with PidfilePinner.default_impl()().pin_pidfile_owner(
+                args.file,
+                timeout=5.,
+        ) as pid:
+            print(pid)
+
+    _main()

{omlish-0.0.0.dev226 → omlish-0.0.0.dev227/omlish.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: omlish
-Version: 0.0.0.dev226
+Version: 0.0.0.dev227
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause

{omlish-0.0.0.dev226 → omlish-0.0.0.dev227}/omlish.egg-info/SOURCES.txt

@@ -517,6 +517,7 @@ omlish/os/temp.py
 omlish/os/pidfiles/__init__.py
 omlish/os/pidfiles/manager.py
 omlish/os/pidfiles/pidfile.py
+omlish/os/pidfiles/pinning.py
 omlish/reflect/__init__.py
 omlish/reflect/inspect.py
 omlish/reflect/ops.py

{omlish-0.0.0.dev226 → omlish-0.0.0.dev227}/pyproject.toml

@@ -12,7 +12,7 @@ authors = [
 urls = {source = 'https://github.com/wrmsr/omlish'}
 license = {text = 'BSD-3-Clause'}
 requires-python = '>=3.12'
-version = '0.0.0.dev226'
+version = '0.0.0.dev227'
 classifiers = [
     'License :: OSI Approved :: BSD License',
     'Development Status :: 2 - Pre-Alpha',