rda-python-dsquasar 3.0.20__tar.gz → 3.0.22__tar.gz

{rda_python_dsquasar-3.0.20/src/rda_python_dsquasar.egg-info → rda_python_dsquasar-3.0.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsquasar
-Version: 3.0.20
+Version: 3.0.22
 Summary: RDA Python package to backup and recover RDA data archives to and from GLOBUS Quasar backup server
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsquasar

{rda_python_dsquasar-3.0.20 → rda_python_dsquasar-3.0.22}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_dsquasar"
-version = "3.0.20"
+version = "3.0.22"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]

{rda_python_dsquasar-3.0.20 → rda_python_dsquasar-3.0.22}/src/rda_python_dsquasar/dsquasar.py

@@ -23,7 +23,6 @@ import os
 import re
 import sys
 import time
-import signal
 from os import path as op
 from time import time as tm
 from rda_python_common.pg_cmd import PgCMD
@@ -124,6 +123,7 @@ class DsQuasar(PgCMD, PgSplit):
          'maxcnt' : 10,
          'dolock' : 1,
          'doemail' : 0,
+         'locflag' : None,  # -L, limit the datasets gathered to one dataset.locflag
          'starttime' : 0,   # wall-clock start of the run, for the PBS walltime guard
          'einfo' : 0,       # set once a progress report is cached into dscheck.einfo
          'eretry' : 0,      # earliest retry time after a progress report failed to cache
@@ -145,7 +145,7 @@ class DsQuasar(PgCMD, PgSplit):
       argv = sys.argv[1:]
       for arg in argv:
          if re.match(r'^-(h|-help)$', arg, re.I): self.show_usage('dsquasar')
-         ms = re.match(r'^-(a|b|c|d|e|E|l|m|n|t|u|w|A|B|D|W)$', arg)
+         ms = re.match(r'^-(a|b|c|d|e|E|l|m|n|t|u|w|A|B|D|L|W)$', arg)
          if ms:
             arg = ms.group(1)
             if  arg == 'b':
@@ -153,7 +153,7 @@ class DsQuasar(PgCMD, PgSplit):
             elif arg in self.sopts:
                self.sopts[arg] = 1
                option = None
-            elif 'AcdlmtwW'.find(arg) > -1:
+            elif 'AcdLlmtwW'.find(arg) > -1:
                option = arg
                if arg == 'd': self.bopts = []
             elif 'BD'.find(arg) > -1:
@@ -180,6 +180,10 @@ class DsQuasar(PgCMD, PgSplit):
                if not (arg == 'Y' or arg == 'N'): self.pglog(arg +": Lock Flag(-l) must be Y or N", self.LGWNEX)
                self.PGBACK['dolock'] = 1 if arg == 'Y' else 0
                option = None
+            elif option == 'L':
+               if not (arg == 'G' or arg == 'O'): self.pglog(arg +": Location Flag(-L) must be G or O", self.LGWNEX)
+               self.PGBACK['locflag'] = arg
+               option = None
             elif option == 'W':
                self.PGBACK['maxworkers'] = int(arg)
                if self.PGBACK['maxworkers'] < 1: self.pglog(arg +": Maximum Workers(-W) must be > 0", self.LGWNEX)
@@ -203,7 +207,7 @@ class DsQuasar(PgCMD, PgSplit):
       self.cmdlog(self.PGBACK['cmd'])
       self.PGBACK['starttime'] = tm()
       self.set_walltime_deadline()
-      self.catch_batch_termination()
+      self.catch_term_signal(self.batch_term_report)
       if self.sopts['u']:
          if self.sopts['a']: self.pglog("-u: Dataset IDs must be provided to Unlock datasets", self.LOGWRN)
          self.unlock_datasets()
@@ -852,42 +856,29 @@ class DsQuasar(PgCMD, PgSplit):
                  self.PGLOG['CURBID'], self.seconds_to_string_time(self.WALLTIME),
                  self.seconds_to_string_time(self.MAXRUNTIME)), self.LOGWRN)
 
-   # PBS kills a job that runs out of walltime with SIGTERM first and SIGKILL a few seconds
-   # later (the MoM's kill_delay, 10 seconds by default), and the same pair is what qdel
-   # sends. SIGTERM is the only warning there is, and nothing in the common library traps
-   # it, so the run used to die on the spot with whatever it had to say unsaid. catching it
-   # turns the last seconds into a report. only a batch run arms this: on the command line
-   # SIGTERM must keep killing the process the way the user expects.
-   def catch_batch_termination(self):
-      if self.PGLOG['CURBID'] < 1: return
-      signal.signal(signal.SIGTERM, self.batch_term_handler)
-
-   # park a progress report and then die of the signal that was sent. this runs on borrowed
-   # time, so it does only what it must: no tar queue counts (two database queries we may
-   # not get to finish - the done count comes from memory), no dataset unlocking (the
-   # dscheck daemon already cleans up after a dead pid). the default handler is restored
-   # first so that a second signal, or the SIGKILL that follows, ends the run outright
-   # instead of re-entering here should the report hang.
+   # park a progress report for a run PBS is about to kill. catch_term_signal() arms this
+   # for a batch run only, restores the default handler before calling it, and re-raises
+   # the signal afterwards, so all this has to do is build the report.
+   # it runs on borrowed time (SIGKILL follows SIGTERM by the MoM's kill_delay, 10 seconds
+   # by default), so it does only what it must: no tar queue counts (two database queries
+   # we may not get to finish - the done count comes from memory), no dataset unlocking
+   # (the dscheck daemon already cleans up after a dead pid).
    # the report may have to be written from inside an interrupted database call; that is
    # why it goes through report_dscheck_email, whose cache_customized_email falls back to
    # sending the mail directly when the UPDATE fails.
-   def batch_term_handler(self, signum, frame):
-      signal.signal(signum, signal.SIG_DFL)
-      # a forked child shares the parent's dscheck record: it must not report for the run
-      if self.PGSIG['PPID'] > 1: os._exit(1)
-      if not self.PGBACK['einfo'] and self.PGBACK['doemail'] and self.PGLOG['DSCHECK']:
-         etime = self.seconds_to_string_time(int(tm() - self.PGBACK['starttime']))
-         amsg = self.action_message()
-         dmsg = self.batch_done_count()
-         wmsg = self.seconds_to_string_time(self.WALLTIME)
-         self.pglog(self.INDENT + "{}: Terminated by signal {} after {} of the {} PBS walltime, {}".format(
-                    amsg, signum, etime, wmsg, dmsg), self.LOGACT)
-         self.set_email("{}: {} terminated after {} of the {} PBS walltime, {}!".format(
-                        self.PGBACK['cmd'], amsg, etime, wmsg, dmsg), self.EMLTOP)
-         title = "dsquasar: {} terminated ({})".format(amsg, dmsg)
-         if self.PGBACK['errcnt']: title += " Error({})".format(self.PGBACK['errcnt'])
-         self.report_dscheck_email(title, 1)
-      os.kill(os.getpid(), signum)
+   def batch_term_report(self, signum, frame):
+      if self.PGBACK['einfo'] or not (self.PGBACK['doemail'] and self.PGLOG['DSCHECK']): return
+      etime = self.seconds_to_string_time(int(tm() - self.PGBACK['starttime']))
+      amsg = self.action_message()
+      dmsg = self.batch_done_count()
+      wmsg = self.seconds_to_string_time(self.WALLTIME)
+      self.pglog(self.INDENT + "{}: Terminated by signal {} after {} of the {} PBS walltime, {}".format(
+                 amsg, signum, etime, wmsg, dmsg), self.LOGACT)
+      self.set_email("{}: {} terminated after {} of the {} PBS walltime, {}!".format(
+                     self.PGBACK['cmd'], amsg, etime, wmsg, dmsg), self.EMLTOP)
+      title = "dsquasar: {} terminated ({})".format(amsg, dmsg)
+      if self.PGBACK['errcnt']: title += " Error({})".format(self.PGBACK['errcnt'])
+      self.report_dscheck_email(title, 1)
 
    # email a report for a run under dscheck control. the progress report is parked in
    # dscheck.einfo and the final report is sent right away. the run holds the check lock
@@ -932,9 +923,10 @@ class DsQuasar(PgCMD, PgSplit):
    # even if PBS kills the job at the walltime. the final report follows and drops the
    # parked one if the run does finish in time. the live email buffers are saved and put
    # back, so the final report still carries everything logged before the cutoff.
-   # PBS sends no catchable warning before the kill - SIGTERM is not trapped and SIGKILL
-   # cannot be - so this parked report is the only thing standing between a killed job and
-   # a run that is never heard from. every action submitted to PBS is guarded, not just the
+   # batch_term_report covers the kill itself, but only the seconds of it: SIGKILL cannot be
+   # caught, and a process stuck in uninterruptible I/O may never run the handler. this parked
+   # report is what stands between a killed job and a run that is never heard from, so it is
+   # kept as the first line of defence. every action submitted to PBS is guarded, not just the
    # tar and transfer ones: -A 16 and the hidden -A 128 are submitted with the same 24 hour
    # walltime and used to be excluded, so they died silently.
    # the tar queue depths are only reported for the actions they describe. both depths are
@@ -1482,13 +1474,29 @@ class DsQuasar(PgCMD, PgSplit):
       fcnt = 0
       # warn per named dataset with no files; stay silent when scanning all datasets
       logact = self.LOGWRN if self.dsids else 0
+      flds = "dsid, backflag, pid, lockhost"
+      # -L splits the work by where the data sits: an Object(O) dataset has to be staged
+      # down from the object store and checksummed before it can be tarred, while a Glade(G)
+      # one is only stat'ed, so the two classes run at wildly different rates and are better
+      # off in separate runs than sharing one walltime.
+      # G is matched as "not O" rather than "= G" on purpose: dataset.locflag is only a
+      # summary of the per-file flags (set_dataset_locflag writes it only when every public
+      # web file agrees, and leaves the old value otherwise), so it can hold a stale value.
+      # as "not O" the two halves stay exhaustive and no dataset falls through both runs
+      lcnd = ''
+      if self.PGBACK['locflag'] == 'O':
+         lcnd = "locflag = 'O'"
+      elif self.PGBACK['locflag']:
+         lcnd = "locflag <> 'O'"
       if self.dsids:
          pgrecs = []
          for dsid in self.dsids:
-            pgrec = self.pgget("dataset", "dsid, backflag, pid", "dsid = '{}'".format(dsid), self.LGWNEX)
+            dcnd = "dsid = '{}'".format(dsid)
+            if lcnd: dcnd += " AND " + lcnd
+            pgrec = self.pgget("dataset", flds, dcnd, self.LGWNEX)
             if pgrec: pgrecs.append(pgrec)
       else:
-         mrecs = self.pgmget("dataset", "dsid, backflag, pid", self.dsid_order().strip(), self.LGWNEX)
+         mrecs = self.pgmget("dataset", flds, (lcnd + self.dsid_order()).strip(), self.LGWNEX)
          dcnt = len(mrecs['dsid']) if mrecs else 0
          pgrecs = [self.onerecord(mrecs, i) for i in range(dcnt)]
       for pgrec in pgrecs:
@@ -1501,7 +1509,16 @@ class DsQuasar(PgCMD, PgSplit):
          # main runs take new files (bid = 0), so it shares no record with the worker holding
          # the lock and must not skip the dataset for it
          if unlock and pgrec['pid'] and not self.PGBACK['chgdays']:
-            if self.lock_dataset(dsid, 0, self.LOGACT) < 1: continue
+            # quiet unlock: a live holder is the normal case for a second worker, and the
+            # library wording for it ("Cannot Unlock") reads like a failure. report the skip
+            # here instead, in terms of the worker that is still on the dataset
+            lstat = self.lock_dataset(dsid, 0, 0)
+            if lstat < 1:
+               # -1 is the live holder; 0 means the record went away or the db errored,
+               # which lock_dataset has already logged as an error
+               if lstat < 0:
+                  self.pglog("{}: skipped, still in use by {}/{}".format(dsid, pgrec['pid'], pgrec['lockhost']), self.LOGACT)
+               continue
          fcnt += self.get_dataset_files(dsid, dsfiles, pgrec['backflag'], logact, sizes)
       if dsfiles:
          s = 's' if fcnt > 1 else ''

{rda_python_dsquasar-3.0.20 → rda_python_dsquasar-3.0.22}/src/rda_python_dsquasar/dsquasar.usg

@@ -8,8 +8,8 @@
  file tarring and transferring is performed through the utility program dsarch.
 
  Usage: dsquasar [-b] [(-a | -t DatasetIDs)] [-c ChangeDays] [-(e|E)]     \
-                 [-(B|D)] [-A ActionBits] [-l (Y|N)] [-m ProcessCount]    \
-                 [-n] [-u] [-d [HostName] [TryCount]] [-h]
+                 [-(B|D)] [-L (G|O)] [-A ActionBits] [-l (Y|N)]           \
+                 [-m ProcessCount] [-n] [-u] [-d [HostName] [TryCount]] [-h]
 
    Either -a or -t must be given (usage is displayed otherwise).  dsquasar runs
    from the Quasar backup work directory; it changes there automatically.
@@ -24,6 +24,21 @@
           '%' is allowed and is expanded against the dataset table.  Ignored,
           with a warning, when -a is also given.
 
+   -L  (G|O)
+          Limit the run by dataset.locflag: O takes the datasets on the object
+          store, G takes everything else.  Applies to -a and to -t alike.  The
+          two classes back up at very different rates: an O dataset has every
+          file staged down from the object store and checksummed before it can
+          be tarred, while a G dataset is only stat'ed.  Running them in one job
+          lets the slow class eat the whole walltime, so give them separate
+          runs, e.g. one submit with -L G and another with -L O.  Defaults to
+          all datasets.
+
+          -L G and -L O together always cover every dataset, so a pair of runs
+          loses nothing.  The split is by dataset, though, and dataset.locflag
+          is only a summary of the per-file flags, so a dataset holding files of
+          both kinds is backed up entirely by whichever run claims it.
+
 
  BACKUP SCOPE
 

{rda_python_dsquasar-3.0.20 → rda_python_dsquasar-3.0.22/src/rda_python_dsquasar.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsquasar
-Version: 3.0.20
+Version: 3.0.22
 Summary: RDA Python package to backup and recover RDA data archives to and from GLOBUS Quasar backup server
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsquasar