rda-python-common 3.0.22__tar.gz → 3.0.24__tar.gz

{rda_python_common-3.0.22/src/rda_python_common.egg-info → rda_python_common-3.0.24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_common
-Version: 3.0.22
+Version: 3.0.24
 Summary: RDA Python common library codes shared by other RDA python packages
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-common
@@ -189,7 +189,7 @@ PgLOG.pglog("hello", PgLOG.LOGWRN)
 python -c "import rda_python_common; print(rda_python_common.__version__)"
 ```
 
-You should see the installed version (currently `3.0.22`).  If the import
+You should see the installed version (currently `3.0.24`).  If the import
 fails, double-check that the active Python environment is the one where you
 ran `pip install`.
 

{rda_python_common-3.0.22 → rda_python_common-3.0.24}/README.md

@@ -165,7 +165,7 @@ PgLOG.pglog("hello", PgLOG.LOGWRN)
 python -c "import rda_python_common; print(rda_python_common.__version__)"
 ```
 
-You should see the installed version (currently `3.0.22`).  If the import
+You should see the installed version (currently `3.0.24`).  If the import
 fails, double-check that the active Python environment is the one where you
 ran `pip install`.
 

{rda_python_common-3.0.22 → rda_python_common-3.0.24}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_common"
-version = "3.0.22"
+version = "3.0.24"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]

{rda_python_common-3.0.22 → rda_python_common-3.0.24}/src/rda_python_common/__init__.py

@@ -22,7 +22,7 @@ object that existing callers expect.
 
 from . import PgLOG, PgUtil, PgDBI, PgFile, PgLock, PgCMD, PgSIG, PgOPT, PgSplit
 
-__version__ = "3.0.22"
+__version__ = "3.0.24"
 
 __all__ = [
    "PgLOG",

{rda_python_common-3.0.22 → rda_python_common-3.0.24}/src/rda_python_common/pg_file.py

@@ -878,6 +878,11 @@ class PgFile(PgUtil, PgSIG):
       objects sharing a key prefix. Each object is downloaded individually and
       keeps its position relative to the prefix under todir.
 
+      A member already staged under todir at the object's size and no older than
+      the object itself is left alone, so a store whose download was cut short
+      (by a batch walltime, say) resumes where it stopped instead of starting
+      over, and a second copy into the same directory costs nothing.
+
       Args:
          todir (str): Destination local directory path.
          fromdir (str): Object key prefix (source path in the bucket).
@@ -890,7 +895,9 @@ class PgFile(PgUtil, PgSIG):
       if not bucket: bucket = self.PGLOG['OBJCTBKT']
       ms = re.match(r'^(.+)/$', fromdir)
       if ms: fromdir = ms.group(1)
-      flist = self.object_glob(fromdir, bucket, 0, logact)
+      # opt 1 adds the LastModified the 'lo' listing already carries; it is only
+      # bits 2 and 8 that would cost a metadata call per key
+      flist = self.object_glob(fromdir, bucket, 1, logact)
       if flist == self.FAILURE: return self.FAILURE
       prefix = fromdir + '/'
       keys = [key for key in flist if key.startswith(prefix)]
@@ -898,8 +905,14 @@ class PgFile(PgUtil, PgSIG):
          return self.lmsg(fromdir, "{}-{} to copy to {}".format(self.OHOST, self.PGLOG['MISSFILE'], todir), logact)
       plen = len(prefix)
       for key in keys:
+         oinfo = flist[key]
          tofile = "{}/{}".format(todir, key[plen:])
-         if not self.object_get_local(tofile, key, flist[key]['data_size'], bucket, logact): return self.FAILURE
+         linfo = self.check_local_file(tofile, 1, logact)
+         # an unparsable LastModified leaves date_modified unset; download again then
+         if linfo and 'date_modified' in oinfo and linfo['data_size'] == oinfo['data_size'] and \
+            self.cmptime(linfo['date_modified'], linfo['time_modified'],
+                         oinfo['date_modified'], oinfo['time_modified']) >= 0: continue
+         if not self.object_get_local(tofile, key, oinfo['data_size'], bucket, logact): return self.FAILURE
       return self.SUCCESS
 
    # Copy a remote file to object

{rda_python_common-3.0.22 → rda_python_common-3.0.24}/src/rda_python_common/pg_sig.py

@@ -63,6 +63,7 @@ class PgSIG(PgDBI):
          'MTIME': 0,    # maximum daemon running time in seconds, 0 for unlimited
          'STIME': 0,    # time the daemon is started
          'STRTM': '',   # string format of 'STIME'
+         'TERMCB': None, # callback to report before a caught termination signal kills the run
       }
 
    # add users for starting this daemon
@@ -480,6 +481,45 @@ class PgSIG(PgDBI):
       if self.PGSIG['PPID'] <= 1 and len(self.CPIDS) > 0:  # passing signal to child processes
          for pid in self.CPIDS: self.kill_process(pid, signum)
 
+   # catch the termination signal so a batch run gets a chance to report before it dies
+   # PBS kills a job that runs out of walltime with SIGTERM first and SIGKILL a few seconds
+   # later (the MoM's kill_delay, 10 seconds by default), and the same pair is what qdel
+   # sends. SIGTERM is the only warning there is, so trapping it turns the last seconds
+   # into a report instead of losing the run silently. only a batch run arms this: on the
+   # command line SIGTERM must keep killing the process the way the user expects.
+   # callback - function called as callback(signum, frame) to build and send the report
+   def catch_term_signal(self, callback):
+      """Trap SIGTERM in a batch run so a report can be sent before the job is killed.
+
+      Does nothing outside a batch run (``PGLOG['CURBID']`` < 1).
+
+      Args:
+         callback: Function called as ``callback(signum, frame)`` from the
+            signal handler to build and send the report.
+      """
+      if self.PGLOG['CURBID'] < 1: return   # only a batch run gets killed off a walltime
+      self.PGSIG['TERMCB'] = callback
+      signal.signal(signal.SIGTERM, self.term_signal_catch)
+
+   # run the registered callback and then die of the signal that was sent
+   def term_signal_catch(self, signum, frame):
+      """Report through the registered callback, then re-raise the signal.
+
+      The default handler is restored before the callback runs so that a
+      second signal, or the SIGKILL that follows, ends the run outright
+      instead of re-entering here should the callback hang.  Re-raising the
+      signal keeps the exit status the one the sender asked for.
+
+      Args:
+         signum (int): The signal number received.
+         frame: The current stack frame, passed on to the callback.
+      """
+      signal.signal(signum, signal.SIG_DFL)
+      callback = self.PGSIG['TERMCB']
+      self.PGSIG['TERMCB'] = None   # report only once
+      if callback: callback(signum, frame)
+      os.kill(os.getpid(), signum)
+
    # wrapper function to call os.kill() logging caught error based on logact
    # return self.SUCCESS is success; PgLog.FAILURE if not
    def kill_process(self, pid, signum, logact=0):
@@ -602,6 +642,9 @@ class PgSIG(PgDBI):
       pid = self.process_fork(msg)
       if pid == 0:  # in child
          signal.signal(signal.SIGQUIT, self.signal_catch)   # catch quit signal only
+         if self.PGSIG['TERMCB']:   # a child shares the parent's check; it must not report
+            signal.signal(signal.SIGTERM, signal.SIG_DFL)
+            self.PGSIG['TERMCB'] = None
          self.PGSIG['PPID'] = self.PGSIG['PID']
          self.PGSIG['PID'] = pid = os.getpid()
          self.cmdlog("Timeout child to " + msg, time.time(), 0)
@@ -693,6 +736,9 @@ class PgSIG(PgDBI):
          self.pglog("{}: starts CPID {} for {}".format(self.PGSIG['DSTR'], pid, pname))
       else:
          signal.signal(signal.SIGQUIT, signal.SIG_DFL)   # turn off catch QUIT signal in child
+         if self.PGSIG['TERMCB']:   # a child shares the parent's check; it must not report
+            signal.signal(signal.SIGTERM, signal.SIG_DFL)
+            self.PGSIG['TERMCB'] = None
          self.PGLOG['LOGMASK'] &= ~self.WARNLG   # turn off warn in child
          self.PGSIG['PPID'] = self.PGSIG['PID']
          self.PGSIG['PID'] = pid = os.getpid()

{rda_python_common-3.0.22 → rda_python_common-3.0.24/src/rda_python_common.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_common
-Version: 3.0.22
+Version: 3.0.24
 Summary: RDA Python common library codes shared by other RDA python packages
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-common
@@ -189,7 +189,7 @@ PgLOG.pglog("hello", PgLOG.LOGWRN)
 python -c "import rda_python_common; print(rda_python_common.__version__)"
 ```
 
-You should see the installed version (currently `3.0.22`).  If the import
+You should see the installed version (currently `3.0.24`).  If the import
 fails, double-check that the active Python environment is the one where you
 ran `pip install`.