omlish 0.0.0.dev226__py3-none-any.whl → 0.0.0.dev227__py3-none-any.whl

omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev226'
-__revision__ = '1f3419dac98e9db37df48a9a952f780d17bf31ed'
+__version__ = '0.0.0.dev227'
+__revision__ = '2faa3e7f9120edd3e6dd79fad6401fd87d89f5cf'
 
 
 #

omlish/diag/lslocks.py

@@ -19,7 +19,7 @@ from ..subprocesses.run import SubprocessRunOutput
 
 
 @dc.dataclass(frozen=True)
-class LsLocksItem:
+class LslocksItem:
     """https://manpages.ubuntu.com/manpages/lunar/man8/lslocks.8.html"""
 
     command: str
@@ -38,7 +38,7 @@ class LsLocksItem:
 
 
 @dc.dataclass(frozen=True)
-class LsLocksCommand(SubprocessRunnable):
+class LslocksCommand(SubprocessRunnable[ta.List[LslocksItem]]):
     pid: ta.Optional[int] = None
     no_inaccessible: bool = False
 
@@ -56,9 +56,9 @@ class LsLocksCommand(SubprocessRunnable):
             stderr='devnull',
         )
 
-    def handle_run_output(self, output: SubprocessRunOutput) -> ta.List[LsLocksItem]:
+    def handle_run_output(self, output: SubprocessRunOutput) -> ta.List[LslocksItem]:
         buf = check.not_none(output.stdout).decode().strip()
         if not buf:
             return []
         obj = json.loads(buf)
-        return unmarshal_obj(obj['locks'], ta.List[LsLocksItem])
+        return unmarshal_obj(obj['locks'], ta.List[LslocksItem])

omlish/diag/lsof.py

@@ -151,7 +151,6 @@ class LsofItem:
 
     @classmethod
     def from_prefixes(cls, dct: ta.Mapping[str, ta.Any]) -> 'LsofItem':
-        print(dct)
         kw: ta.Dict[str, ta.Any] = {
             fld.name: val
             for pfx, val in dct.items()
@@ -216,7 +215,8 @@ LsofItem._DEFAULT_PREFIXES = ''.join(LsofItem._FIELDS_BY_PREFIX)  # noqa
 
 
 @dc.dataclass(frozen=True)
-class LsofCommand(SubprocessRunnable[ta.List[LsofItem]]):
+class \
+        LsofCommand(SubprocessRunnable[ta.List[LsofItem]]):
     pid: ta.Optional[int] = None
     file: ta.Optional[str] = None
 

omlish/os/pidfiles/manager.py

@@ -15,7 +15,10 @@ from .pidfile import Pidfile
 
 class _PidfileManager:
     """
-    Manager for controlled inheritance of Pidfiles across forks.
+    Manager for controlled inheritance of Pidfiles across forks in the presence of multiple threads. There is of course
+    no safe or correct way to mix the use of fork and multiple active threads, and one should never write code which
+    does so, but in the Real World one may still find oneself in such a situation outside of their control (such as when
+    running under Pycharm's debugger which forces the use of forked multiprocessing).
 
     Not implemented as an instantiated class as there is no way to unregister at_fork listeners, and because Pidfiles
     may be pickled and there must be no possibility of accidentally unpickling and instantiating a new instance of the

omlish/os/pidfiles/pidfile.py

@@ -109,7 +109,7 @@ class Pidfile:
         del self._f
         return True
 
-    def try_lock(self) -> bool:
+    def try_acquire_lock(self) -> bool:
         try:
             fcntl.flock(self._f, fcntl.LOCK_EX | fcntl.LOCK_NB)
             return True
@@ -117,14 +117,29 @@ class Pidfile:
         except OSError:
             return False
 
-    def ensure_locked(self) -> None:
-        if not self.try_lock():
-            raise RuntimeError('Could not get lock')
+    #
+
+    class Error(Exception):
+        pass
+
+    class LockedError(Error):
+        pass
+
+    def acquire_lock(self) -> None:
+        if not self.try_acquire_lock():
+            raise self.LockedError
+
+    class NotLockedError(Error):
+        pass
+
+    def ensure_cannot_lock(self) -> None:
+        if self.try_acquire_lock():
+            raise self.NotLockedError
 
     #
 
     def write(self, pid: ta.Optional[int] = None) -> None:
-        self.ensure_locked()
+        self.acquire_lock()
 
         if pid is None:
             pid = os.getpid()
@@ -135,18 +150,23 @@ class Pidfile:
         self._f.flush()
 
     def clear(self) -> None:
-        self.ensure_locked()
+        self.acquire_lock()
 
         self._f.seek(0)
         self._f.truncate()
 
-    def read(self) -> int:
-        if self.try_lock():
-            raise RuntimeError('Got lock')
+    #
+
+    def read(self) -> ta.Optional[int]:
+        self.ensure_cannot_lock()
 
         self._f.seek(0)
-        return int(self._f.read())
+        buf = self._f.read()
+        if not buf:
+            return None
+        return int(buf)
 
     def kill(self, sig: int = signal.SIGTERM) -> None:
-        pid = self.read()
+        if (pid := self.read()) is None:
+            raise self.Error(f'Pidfile locked but empty')
         os.kill(pid, sig)

omlish/os/pidfiles/pinning.py

@@ -0,0 +1,246 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+"""
+Strategies:
+ - linux
+  - get pid of owner (lslocks or F_GETLK)
+  - open pidfd to owner pid
+  - re-check pid of owner
+ - darwin
+  - get pids of referrers (lsof)
+  - read pid from file
+  - ensure pid is in referrers
+  - optionally loop
+  - ? setup pid death watcher? still a race
+"""
+import abc
+import contextlib
+import os.path
+import shutil
+import sys
+import time
+import typing as ta
+
+from ...diag.lslocks import LslocksCommand
+from ...diag.lsof import LsofCommand
+from ...lite.timeouts import Timeout
+from ...lite.timeouts import TimeoutLike
+from ...subprocesses.sync import subprocesses  # noqa
+from .pidfile import Pidfile
+
+
+##
+
+
+class PidfilePinner(abc.ABC):
+    def __init__(
+            self,
+            *,
+            sleep_s: float = .1,
+    ) -> None:
+        super().__init__()
+
+        self._sleep_s = sleep_s
+
+    @classmethod
+    @abc.abstractmethod
+    def is_available(cls) -> bool:
+        raise NotImplementedError
+
+    class NoOwnerError(Exception):
+        pass
+
+    @abc.abstractmethod
+    def _pin_pidfile_owner(self, pidfile: Pidfile, timeout: Timeout) -> ta.ContextManager[int]:
+        raise NotImplementedError
+
+    @contextlib.contextmanager
+    def pin_pidfile_owner(
+            self,
+            path: str,
+            *,
+            timeout: ta.Optional[TimeoutLike] = None,
+    ) -> ta.Iterator[int]:
+        timeout = Timeout.of(timeout)
+
+        if not os.path.isfile(path):
+            raise self.NoOwnerError
+
+        with Pidfile(path, inheritable=False) as pf:
+            try:
+                with self._pin_pidfile_owner(pf, timeout) as pid:
+                    yield pid
+
+            except Pidfile.NotLockedError:
+                raise self.NoOwnerError from None
+
+    @classmethod
+    def default_impl(cls) -> ta.Type['PidfilePinner']:
+        for impl in [
+            LslocksPidfdPidfilePinner,
+            LsofPidfilePinner,
+        ]:
+            if impl.is_available():
+                return impl
+        return UnverifiedPidfilePinner
+
+
+##
+
+
+class UnverifiedPidfilePinner(PidfilePinner):
+    @classmethod
+    def is_available(cls) -> bool:
+        return True
+
+    @contextlib.contextmanager
+    def _pin_pidfile_owner(self, pidfile: Pidfile, timeout: Timeout) -> ta.Iterator[int]:
+        while (pid := pidfile.read()) is None:
+            time.sleep(self._sleep_s)
+            timeout()
+
+        yield pid
+
+
+##
+
+
+class LsofPidfilePinner(PidfilePinner):
+    """
+    Fundamentally wrong, but still better than nothing. Simply reads the file contents and ensures a valid contained pid
+    has the file open via `lsof`.
+    """
+
+    @classmethod
+    def is_available(cls) -> bool:
+        return shutil.which('lsof') is not None
+
+    def _try_read_and_verify(self, pf: Pidfile, timeout: Timeout) -> ta.Optional[int]:
+        if (initial_pid := pf.read()) is None:
+            return None
+
+        lsof_output = LsofCommand(
+            # pid=initial_pid,
+            file=os.path.abspath(pf.path),
+        ).run(
+            timeout=timeout,
+        )
+
+        lsof_pids: ta.Set[int] = set()
+        for li in lsof_output:
+            if li.pid is None:
+                continue
+            try:
+                li_pid = int(li.pid)
+            except ValueError:
+                continue
+            lsof_pids.add(li_pid)
+
+        if initial_pid not in lsof_pids:
+            return None
+
+        if (reread_pid := pf.read()) is None or reread_pid != initial_pid:
+            return None
+
+        return reread_pid
+
+    @contextlib.contextmanager
+    def _pin_pidfile_owner(self, pidfile: Pidfile, timeout: Timeout) -> ta.Iterator[int]:
+        while (pid := self._try_read_and_verify(pidfile, timeout)) is None:
+            time.sleep(self._sleep_s)
+            timeout()
+
+        yield pid
+
+
+##
+
+
+class LslocksPidfdPidfilePinner(PidfilePinner):
+    """
+    Finds the locking pid via `lslocks`, opens a pidfd, then re-runs `lslocks` and rechecks the locking pid is the same.
+    """
+
+    @classmethod
+    def is_available(cls) -> bool:
+        return sys.platform == 'linux' and shutil.which('lslocks') is not None
+
+    def _read_locking_pid(self, path: str, timeout: Timeout) -> int:
+        lsl_output = LslocksCommand().run(timeout=timeout)
+
+        lsl_pids = {
+            li.pid
+            for li in lsl_output
+            if li.path == path
+            and li.type == 'FLOCK'
+        }
+        if not lsl_pids:
+            raise self.NoOwnerError
+        if len(lsl_pids) != 1:
+            raise RuntimeError(f'Multiple locks on file: {path}')
+
+        [pid] = lsl_pids
+        return pid
+
+    class _Result(ta.NamedTuple):
+        pid: int
+        pidfd: int
+
+    def _try_read_and_verify(
+            self,
+            pidfile: Pidfile,
+            timeout: Timeout,
+    ) -> ta.Optional[_Result]:
+        path = os.path.abspath(pidfile.path)
+        initial_pid = self._read_locking_pid(path, timeout)
+
+        try:
+            pidfd = os.open(f'/proc/{initial_pid}', os.O_RDONLY)
+        except FileNotFoundError:
+            raise self.NoOwnerError from None
+
+        try:
+            reread_pid = self._read_locking_pid(path, timeout)
+            if reread_pid != initial_pid:
+                os.close(pidfd)
+                return None
+
+            return self._Result(initial_pid, pidfd)
+
+        except BaseException:
+            os.close(pidfd)
+            raise
+
+    @contextlib.contextmanager
+    def _pin_pidfile_owner(
+            self,
+            pidfile: Pidfile,
+            timeout: Timeout,
+    ) -> ta.Iterator[int]:
+        while (res := self._try_read_and_verify(pidfile, timeout)) is None:
+            time.sleep(self._sleep_s)
+            timeout()
+
+        try:
+            yield res.pid
+        finally:
+            os.close(res.pidfd)
+
+
+##
+
+
+if __name__ == '__main__':
+    def _main() -> None:
+        argparse = __import__('argparse')
+        parser = argparse.ArgumentParser()
+        parser.add_argument('file')
+        args = parser.parse_args()
+
+        with PidfilePinner.default_impl()().pin_pidfile_owner(
+                args.file,
+                timeout=5.,
+        ) as pid:
+            print(pid)
+
+    _main()

{omlish-0.0.0.dev226.dist-info → omlish-0.0.0.dev227.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: omlish
-Version: 0.0.0.dev226
+Version: 0.0.0.dev227
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause

{omlish-0.0.0.dev226.dist-info → omlish-0.0.0.dev227.dist-info}/RECORD

@@ -1,5 +1,5 @@
 omlish/.manifests.json,sha256=YGmAnUBszmosQQ_7Hh2wwtDiYdYZ4unNKYzOtALuels,7968
-omlish/__about__.py,sha256=QR1QmjxqAJ538XauJdEuHH505rJ9QF67p3blNgi6kwc,3380
+omlish/__about__.py,sha256=F8eB77QCwYkeNA3ZSyaAdpFyPxSptO9EjeuJvnM2Lrg,3380
 omlish/__init__.py,sha256=SsyiITTuK0v74XpKV8dqNaCmjOlan1JZKrHQv5rWKPA,253
 omlish/c3.py,sha256=ubu7lHwss5V4UznbejAI0qXhXahrU01MysuHOZI9C4U,8116
 omlish/cached.py,sha256=UI-XTFBwA6YXWJJJeBn-WkwBkfzDjLBBaZf4nIJA9y0,510
@@ -203,8 +203,8 @@ omlish/dataclasses/impl/utils.py,sha256=aER2iL3UAtgS1BdLuEvTr9Tr2wC28wk1kiOeO-jI
 omlish/diag/__init__.py,sha256=4S8v0myJM4Zld6_FV6cPe_nSv0aJb6kXftEit0HkiGE,1141
 omlish/diag/asts.py,sha256=BveUUNUcaAm4Hg55f4ZxGSI313E4L8cCZ5XjHpEkKVI,3325
 omlish/diag/debug.py,sha256=ClED7kKXeVMyKrjGIxcq14kXk9kvUJfytBQwK9y7c4Q,1637
-omlish/diag/lslocks.py,sha256=7I_u-Vx0Dz0KN59Y6dk9nBZynHa6usvKIXVSywGCiWE,1722
-omlish/diag/lsof.py,sha256=gMtNmfBo1AlYmvZWBB51u4NfbfC6yJyEVwfr1HgktP8,9328
+omlish/diag/lslocks.py,sha256=fWI3SZwgEkhipVfSqvzVzREJRShcDYmlYByHBT0LToc,1744
+omlish/diag/lsof.py,sha256=-s-0QKFC-_vfdkxhk0m_Lq-PVrXCY7TUFV9ff_Q_gdQ,9319
 omlish/diag/procfs.py,sha256=KaGTAA2Gj8eEEp7MjClRe4aimwzd-HDABThFzvq2cBQ,9684
 omlish/diag/procstats.py,sha256=UkqxREqfd-38xPYZ9T1SIJISz5ARQCEhTtOZrxtm2dE,777
 omlish/diag/ps.py,sha256=mwraPO6Nze_2r9RiC4dyLL0aJ_lHR1_fSN9BlYNBoHw,1651
@@ -505,8 +505,9 @@ omlish/os/paths.py,sha256=hqPiyg_eYaRoIVPdAeX4oeLEV4Kpln_XsH0tHvbOf8Q,844
 omlish/os/sizes.py,sha256=ohkALLvqSqBX4iR-7DMKJ4pfOCRdZXV8htH4QywUNM0,152
 omlish/os/temp.py,sha256=P97KiVeNB7rfGn4tlgU5ro86JUxAsiphLMlxsjQgfB0,1198
 omlish/os/pidfiles/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-omlish/os/pidfiles/manager.py,sha256=ssnxCvSoL3OapzcMmxSgiy9o1NogR6PTX4LLTJXuPkM,2830
-omlish/os/pidfiles/pidfile.py,sha256=833aPCV7cQqFqYnxwma8np0eoy0CJzVHRskn-YSQwis,3378
+omlish/os/pidfiles/manager.py,sha256=XP_0ynjWDG2wjRN3sIKjmFafpf-er3AYm8E7aXs16fA,3209
+omlish/os/pidfiles/pidfile.py,sha256=WmZt_c8fvztPgZQnYHhcQCKWgHqAEsaI3Ggz6Wqgkc8,3748
+omlish/os/pidfiles/pinning.py,sha256=YmlHK4G4t38Uldx-kHQBSMSwCzETevDYBDED9HhCCvQ,6153
 omlish/reflect/__init__.py,sha256=JBWwxKwP4IEaomkK0PTju02STU1BVXT14SCrShT1Sm0,769
 omlish/reflect/inspect.py,sha256=veJ424-9oZrqyvhVpvxOi7hcKW-PDBkdYL2yjrFlk4o,495
 omlish/reflect/ops.py,sha256=RJ6jzrM4ieFsXzWyNXWV43O_WgzEaUvlHSc5N2ezW2A,2044
@@ -679,9 +680,9 @@ omlish/text/indent.py,sha256=YjtJEBYWuk8--b9JU_T6q4yxV85_TR7VEVr5ViRCFwk,1336
 omlish/text/minja.py,sha256=jZC-fp3Xuhx48ppqsf2Sf1pHbC0t8XBB7UpUUoOk2Qw,5751
 omlish/text/parts.py,sha256=7vPF1aTZdvLVYJ4EwBZVzRSy8XB3YqPd7JwEnNGGAOo,6495
 omlish/text/random.py,sha256=jNWpqiaKjKyTdMXC-pWAsSC10AAP-cmRRPVhm59ZWLk,194
-omlish-0.0.0.dev226.dist-info/LICENSE,sha256=B_hVtavaA8zCYDW99DYdcpDLKz1n3BBRjZrcbv8uG8c,1451
-omlish-0.0.0.dev226.dist-info/METADATA,sha256=0HcF368bfjlzkGzprdbRX4GJagL5c9PDPRUqiBYWIlY,4176
-omlish-0.0.0.dev226.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-omlish-0.0.0.dev226.dist-info/entry_points.txt,sha256=Lt84WvRZJskWCAS7xnQGZIeVWksprtUHj0llrvVmod8,35
-omlish-0.0.0.dev226.dist-info/top_level.txt,sha256=pePsKdLu7DvtUiecdYXJ78iO80uDNmBlqe-8hOzOmfs,7
-omlish-0.0.0.dev226.dist-info/RECORD,,
+omlish-0.0.0.dev227.dist-info/LICENSE,sha256=B_hVtavaA8zCYDW99DYdcpDLKz1n3BBRjZrcbv8uG8c,1451
+omlish-0.0.0.dev227.dist-info/METADATA,sha256=Pl1bZYBMsE5uKxtT265EkiV61cZ6y6-u11m6uqsXwz4,4176
+omlish-0.0.0.dev227.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+omlish-0.0.0.dev227.dist-info/entry_points.txt,sha256=Lt84WvRZJskWCAS7xnQGZIeVWksprtUHj0llrvVmod8,35
+omlish-0.0.0.dev227.dist-info/top_level.txt,sha256=pePsKdLu7DvtUiecdYXJ78iO80uDNmBlqe-8hOzOmfs,7
+omlish-0.0.0.dev227.dist-info/RECORD,,