rda-python-common 3.0.28__tar.gz → 3.0.30__tar.gz

{rda_python_common-3.0.28/src/rda_python_common.egg-info → rda_python_common-3.0.30}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_common
-Version: 3.0.28
+Version: 3.0.30
 Summary: RDA Python common library codes shared by other RDA python packages
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-common
@@ -189,7 +189,7 @@ PgLOG.pglog("hello", PgLOG.LOGWRN)
 python -c "import rda_python_common; print(rda_python_common.__version__)"
 ```
 
-You should see the installed version (currently `3.0.28`).  If the import
+You should see the installed version (currently `3.0.30`).  If the import
 fails, double-check that the active Python environment is the one where you
 ran `pip install`.
 

{rda_python_common-3.0.28 → rda_python_common-3.0.30}/README.md

@@ -165,7 +165,7 @@ PgLOG.pglog("hello", PgLOG.LOGWRN)
 python -c "import rda_python_common; print(rda_python_common.__version__)"
 ```
 
-You should see the installed version (currently `3.0.28`).  If the import
+You should see the installed version (currently `3.0.30`).  If the import
 fails, double-check that the active Python environment is the one where you
 ran `pip install`.
 

{rda_python_common-3.0.28 → rda_python_common-3.0.30}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_common"
-version = "3.0.28"
+version = "3.0.30"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]

{rda_python_common-3.0.28 → rda_python_common-3.0.30}/src/rda_python_common/__init__.py

@@ -22,7 +22,7 @@ object that existing callers expect.
 
 from . import PgLOG, PgUtil, PgDBI, PgFile, PgLock, PgCMD, PgSIG, PgOPT, PgSplit
 
-__version__ = "3.0.28"
+__version__ = "3.0.30"
 
 __all__ = [
    "PgLOG",

{rda_python_common-3.0.28 → rda_python_common-3.0.30}/src/rda_python_common/pg_sig.py

@@ -13,6 +13,7 @@ import re
 import sys
 import errno
 import signal
+import select
 import time
 import subprocess
 import psutil
@@ -52,7 +53,12 @@ class PgSIG(PgDBI):
          'MPROC': 1,    # default number of multiple processes
          'BPROC': 1,    # default number of multiple background processes
          'ETIME': 20,   # default error waiting time (in seconds)
-         'WTIME': 120,  # default waiting time (in seconds)
+         'WTIME': 120,  # default waiting time (in seconds), the cap of an adaptive wait
+         'WTMIN': 5,    # shortest adaptive wait (in seconds) for child/background processes
+         'WTINC': 5,    # seconds added to the adaptive wait after each loop still waiting
+         'CWTIME': 0,   # current adaptive wait (in seconds), 0 before the first wait
+         'WLOOP': -1,   # loop count of the last adaptive wait sleep
+         'WPIPE': None, # (read fd, write fd) of the pipe SIGCHLD writes to wake up a wait
          'DTIME': 600,  # the daemon record refresh time (in seconds)
          'RTIME': 2400, # the web rda config unlocking and unconfigured system down waiting time (in seconds)
          'CTIME': 4800, # the lock cleaning & configued system down waiting time (in seconds)
@@ -154,6 +160,7 @@ class PgSIG(PgDBI):
       os.umask(0)
       # setup to catch signals in daemon only
       signal.signal(signal.SIGCHLD, self.clean_dead_child)
+      self.open_wake_pipe()
       signal.signal(signal.SIGQUIT, self.signal_catch)
       signal.signal(signal.SIGUSR1, self.signal_catch)
       signal.signal(signal.SIGUSR2, self.signal_catch)
@@ -556,7 +563,7 @@ class PgSIG(PgDBI):
          signum (int): The signal number received (expected to be SIGCHLD).
          frame: The current stack frame (unused).
       """
-      live = 0
+      live = wake = 0
       while True:
          try:
             dpid, status = os.waitpid(-1, os.WNOHANG)
@@ -570,7 +577,57 @@ class PgSIG(PgDBI):
               if live > 0: break   # wait twice if a process is still a live
               live += 1
             elif self.PGSIG['PPID'] < 2:
-               if dpid in self.CPIDS: del self.CPIDS[dpid]
+               if dpid in self.CPIDS:
+                  del self.CPIDS[dpid]
+                  wake = 1
+      if wake: self.wake_waiter()   # a managed child is done; end any wait for it now
+
+   # open the pipe a SIGCHLD handler writes to, so a wait for processes wakes up at once
+   def open_wake_pipe(self):
+      """Open the non-blocking self-pipe that wakes ``show_wait_message()`` on SIGCHLD.
+
+      ``clean_dead_child()`` writes one byte to it when a child process in ``CPIDS``
+      is done, and ``show_wait_message()`` waits on it with the adaptive wait time as
+      the timeout, so a free process slot is used at once instead of after a sleep.
+      """
+      if self.PGSIG['WPIPE']: return
+      try:
+         rfd, wfd = os.pipe()
+         os.set_blocking(rfd, False)
+         os.set_blocking(wfd, False)
+      except OSError as e:
+         self.pglog("Cannot open wake pipe, wait by sleeping: {}".format(str(e)), self.LOGWRN)
+         return
+      self.PGSIG['WPIPE'] = (rfd, wfd)
+
+   # close the wake pipe, such as in a child process that inherited it
+   def close_wake_pipe(self):
+      """Close the SIGCHLD wake pipe and forget it."""
+      if not self.PGSIG['WPIPE']: return
+      for fd in self.PGSIG['WPIPE']:
+         try:
+            os.close(fd)
+         except OSError:
+            pass
+      self.PGSIG['WPIPE'] = None
+
+   # write one byte to the wake pipe to end a wait in show_wait_message()
+   def wake_waiter(self):
+      """Wake a wait in ``show_wait_message()``; a full pipe already wakes it."""
+      if not self.PGSIG['WPIPE']: return
+      try:
+         os.write(self.PGSIG['WPIPE'][1], b'x')
+      except OSError:
+         pass   # pipe full (a wake is pending already) or closed
+
+   # read out all pending wake bytes, so a past wake does not end the next wait
+   def drain_wake_pipe(self):
+      """Read out any pending bytes of the SIGCHLD wake pipe."""
+      if not self.PGSIG['WPIPE']: return
+      try:
+         while os.read(self.PGSIG['WPIPE'][0], 512): pass
+      except OSError:
+         pass   # BlockingIOError when empty
 
    # send signal to daemon and exit
    def signal_daemon(self, sname, aname, uname):
@@ -744,6 +801,7 @@ class PgSIG(PgDBI):
          self.PGSIG['PID'] = pid = os.getpid()
          self.PGSIG['MPROC'] = 1   # 1 in child process
          self.CBIDS = {}         # empty backgroud proces info in case not
+         self.close_wake_pipe()  # the wake pipe is the parent's
          self.PGSIG['DSTR'] += ": CPID {} for {}".format(pid, pname)
          self.cmdlog("CPID {} for {}".format(pid, pname))
          self.pgdisconnect(0)  # disconnect database in child
@@ -789,6 +847,7 @@ class PgSIG(PgDBI):
       if logact&self.EXITLG: logact &= ~self.EXITLG
       ccnt = i = 0
       if dowait < 0: ccnt = 1 if (pid or pname) else self.PGSIG['MPROC']
+      self.drain_wake_pipe()   # the check below covers any child done before now
       while True:
          pcnt = 0
          if not pid and pname: pid = self.pname2cpid(pname)
@@ -841,6 +900,7 @@ class PgSIG(PgDBI):
          self.check_vuser(uname, aname)
       signal.signal(signal.SIGQUIT, self.signal_catch)   # catch quit signal only
       signal.signal(signal.SIGCHLD, self.clean_dead_child)
+      self.open_wake_pipe()
       self.PGSIG['DSTR'] = dstr
       self.PGSIG['DNAME'] = aname
       self.PGSIG['PPID'] = 0
@@ -1281,21 +1341,64 @@ class PgSIG(PgDBI):
       if wtime: time.sleep(wtime)
       return wtime
 
-   # show wait message every dintv and then sleep for PGSIG['WTIME']
+   # show wait message every 30 loops and then sleep for an adaptive wait time
    def show_wait_message(self, loop, msg, logact=None, dowait=0):
       """Log a wait message every 30 loops and optionally sleep.
 
       Args:
-         loop (int): Current loop iteration count.
+         loop (int): Current loop iteration count, 0 for the first sleep of a wait.
          msg (str): Message to log.
          logact (int, optional): Log action flags. Defaults to ``LOGWRN``.
-         dowait (int, optional): If non-zero, sleep for ``PGSIG['WTIME']``
-            seconds. Defaults to 0.
+         dowait (int, optional): If non-zero, wait for the adaptive wait time
+            from ``adaptive_wait_time()``, at most ``PGSIG['WTIME']`` seconds,
+            or until a child process in ``CPIDS`` is done, whichever is first
+            (see ``open_wake_pipe()``).  Defaults to 0.
       """
       if logact is None: logact = self.LOGWRN
       if loop > 0 and (loop%30) == 0:
          self.pglog("{} at {}".format(msg, self.current_datetime()), logact)
-      if dowait: time.sleep(self.PGSIG['WTIME'])
+      if not dowait: return
+      wtime = self.adaptive_wait_time(loop)
+      if self.PGSIG['WPIPE'] and self.PGSIG['PPID'] < 2:
+         # wake as soon as a child is done; the adaptive time is only the longest wait
+         rfd = self.PGSIG['WPIPE'][0]
+         try:
+            select.select([rfd], [], [], wtime)
+         except (OSError, ValueError):
+            time.sleep(wtime)
+         self.drain_wake_pipe()
+      else:
+         time.sleep(wtime)
+
+   # get the adaptive time to sleep for one loop of a wait for processes
+   def adaptive_wait_time(self, loop):
+      """Return how long to sleep for one loop of a wait for child/background processes.
+
+      The wait starts at ``PGSIG['WTMIN']`` seconds and grows by ``PGSIG['WTINC']``
+      after each loop that still finds no process done, capped at the polling time
+      ``PGSIG['WTIME']``, so long-running processes are not checked too often while
+      short ones are not waited on for the full polling time.  The current wait
+      carries over to the next wait; it goes back to ``WTMIN`` when the previous
+      wait was over after its first sleep, since the processes are short.  Set
+      ``PGSIG['WTMIN']`` to ``WTIME`` or larger for the old fixed wait.
+
+      Args:
+         loop (int): Loop count of the current wait, 0 for its first sleep.
+
+      Returns:
+         int: Seconds to sleep.
+      """
+      wmax = self.PGSIG['WTIME']
+      wmin = min(self.PGSIG['WTMIN'], wmax)
+      wtime = self.PGSIG['CWTIME']
+      if loop == 0:
+         if not wtime or self.PGSIG['WLOOP'] == 0: wtime = wmin
+      else:
+         wtime += self.PGSIG['WTINC']
+      wtime = max(wmin, min(wtime, wmax))
+      self.PGSIG['CWTIME'] = wtime
+      self.PGSIG['WLOOP'] = loop
+      return wtime
 
    # register a time out function to raise a time out error
    @contextmanager

{rda_python_common-3.0.28 → rda_python_common-3.0.30/src/rda_python_common.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_common
-Version: 3.0.28
+Version: 3.0.30
 Summary: RDA Python common library codes shared by other RDA python packages
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-common
@@ -189,7 +189,7 @@ PgLOG.pglog("hello", PgLOG.LOGWRN)
 python -c "import rda_python_common; print(rda_python_common.__version__)"
 ```
 
-You should see the installed version (currently `3.0.28`).  If the import
+You should see the installed version (currently `3.0.30`).  If the import
 fails, double-check that the active Python environment is the one where you
 ran `pip install`.