rda-python-dsquasar 3.0.21__tar.gz → 3.0.22__tar.gz

{rda_python_dsquasar-3.0.21/src/rda_python_dsquasar.egg-info → rda_python_dsquasar-3.0.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsquasar
-Version: 3.0.21
+Version: 3.0.22
 Summary: RDA Python package to backup and recover RDA data archives to and from GLOBUS Quasar backup server
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsquasar

{rda_python_dsquasar-3.0.21 → rda_python_dsquasar-3.0.22}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_dsquasar"
-version = "3.0.21"
+version = "3.0.22"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]

{rda_python_dsquasar-3.0.21 → rda_python_dsquasar-3.0.22}/src/rda_python_dsquasar/dsquasar.py

@@ -123,6 +123,7 @@ class DsQuasar(PgCMD, PgSplit):
          'maxcnt' : 10,
          'dolock' : 1,
          'doemail' : 0,
+         'locflag' : None,  # -L, limit the datasets gathered to one dataset.locflag
          'starttime' : 0,   # wall-clock start of the run, for the PBS walltime guard
          'einfo' : 0,       # set once a progress report is cached into dscheck.einfo
          'eretry' : 0,      # earliest retry time after a progress report failed to cache
@@ -144,7 +145,7 @@ class DsQuasar(PgCMD, PgSplit):
       argv = sys.argv[1:]
       for arg in argv:
          if re.match(r'^-(h|-help)$', arg, re.I): self.show_usage('dsquasar')
-         ms = re.match(r'^-(a|b|c|d|e|E|l|m|n|t|u|w|A|B|D|W)$', arg)
+         ms = re.match(r'^-(a|b|c|d|e|E|l|m|n|t|u|w|A|B|D|L|W)$', arg)
          if ms:
             arg = ms.group(1)
             if  arg == 'b':
@@ -152,7 +153,7 @@ class DsQuasar(PgCMD, PgSplit):
             elif arg in self.sopts:
                self.sopts[arg] = 1
                option = None
-            elif 'AcdlmtwW'.find(arg) > -1:
+            elif 'AcdLlmtwW'.find(arg) > -1:
                option = arg
                if arg == 'd': self.bopts = []
             elif 'BD'.find(arg) > -1:
@@ -179,6 +180,10 @@ class DsQuasar(PgCMD, PgSplit):
                if not (arg == 'Y' or arg == 'N'): self.pglog(arg +": Lock Flag(-l) must be Y or N", self.LGWNEX)
                self.PGBACK['dolock'] = 1 if arg == 'Y' else 0
                option = None
+            elif option == 'L':
+               if not (arg == 'G' or arg == 'O'): self.pglog(arg +": Location Flag(-L) must be G or O", self.LGWNEX)
+               self.PGBACK['locflag'] = arg
+               option = None
             elif option == 'W':
                self.PGBACK['maxworkers'] = int(arg)
                if self.PGBACK['maxworkers'] < 1: self.pglog(arg +": Maximum Workers(-W) must be > 0", self.LGWNEX)
@@ -918,9 +923,10 @@ class DsQuasar(PgCMD, PgSplit):
    # even if PBS kills the job at the walltime. the final report follows and drops the
    # parked one if the run does finish in time. the live email buffers are saved and put
    # back, so the final report still carries everything logged before the cutoff.
-   # PBS sends no catchable warning before the kill - SIGTERM is not trapped and SIGKILL
-   # cannot be - so this parked report is the only thing standing between a killed job and
-   # a run that is never heard from. every action submitted to PBS is guarded, not just the
+   # batch_term_report covers the kill itself, but only the seconds of it: SIGKILL cannot be
+   # caught, and a process stuck in uninterruptible I/O may never run the handler. this parked
+   # report is what stands between a killed job and a run that is never heard from, so it is
+   # kept as the first line of defence. every action submitted to PBS is guarded, not just the
    # tar and transfer ones: -A 16 and the hidden -A 128 are submitted with the same 24 hour
    # walltime and used to be excluded, so they died silently.
    # the tar queue depths are only reported for the actions they describe. both depths are
@@ -1468,13 +1474,29 @@ class DsQuasar(PgCMD, PgSplit):
       fcnt = 0
       # warn per named dataset with no files; stay silent when scanning all datasets
       logact = self.LOGWRN if self.dsids else 0
+      flds = "dsid, backflag, pid, lockhost"
+      # -L splits the work by where the data sits: an Object(O) dataset has to be staged
+      # down from the object store and checksummed before it can be tarred, while a Glade(G)
+      # one is only stat'ed, so the two classes run at wildly different rates and are better
+      # off in separate runs than sharing one walltime.
+      # G is matched as "not O" rather than "= G" on purpose: dataset.locflag is only a
+      # summary of the per-file flags (set_dataset_locflag writes it only when every public
+      # web file agrees, and leaves the old value otherwise), so it can hold a stale value.
+      # as "not O" the two halves stay exhaustive and no dataset falls through both runs
+      lcnd = ''
+      if self.PGBACK['locflag'] == 'O':
+         lcnd = "locflag = 'O'"
+      elif self.PGBACK['locflag']:
+         lcnd = "locflag <> 'O'"
       if self.dsids:
          pgrecs = []
          for dsid in self.dsids:
-            pgrec = self.pgget("dataset", "dsid, backflag, pid", "dsid = '{}'".format(dsid), self.LGWNEX)
+            dcnd = "dsid = '{}'".format(dsid)
+            if lcnd: dcnd += " AND " + lcnd
+            pgrec = self.pgget("dataset", flds, dcnd, self.LGWNEX)
             if pgrec: pgrecs.append(pgrec)
       else:
-         mrecs = self.pgmget("dataset", "dsid, backflag, pid", self.dsid_order().strip(), self.LGWNEX)
+         mrecs = self.pgmget("dataset", flds, (lcnd + self.dsid_order()).strip(), self.LGWNEX)
          dcnt = len(mrecs['dsid']) if mrecs else 0
          pgrecs = [self.onerecord(mrecs, i) for i in range(dcnt)]
       for pgrec in pgrecs:
@@ -1487,7 +1509,16 @@ class DsQuasar(PgCMD, PgSplit):
          # main runs take new files (bid = 0), so it shares no record with the worker holding
          # the lock and must not skip the dataset for it
          if unlock and pgrec['pid'] and not self.PGBACK['chgdays']:
-            if self.lock_dataset(dsid, 0, self.LOGACT) < 1: continue
+            # quiet unlock: a live holder is the normal case for a second worker, and the
+            # library wording for it ("Cannot Unlock") reads like a failure. report the skip
+            # here instead, in terms of the worker that is still on the dataset
+            lstat = self.lock_dataset(dsid, 0, 0)
+            if lstat < 1:
+               # -1 is the live holder; 0 means the record went away or the db errored,
+               # which lock_dataset has already logged as an error
+               if lstat < 0:
+                  self.pglog("{}: skipped, still in use by {}/{}".format(dsid, pgrec['pid'], pgrec['lockhost']), self.LOGACT)
+               continue
          fcnt += self.get_dataset_files(dsid, dsfiles, pgrec['backflag'], logact, sizes)
       if dsfiles:
          s = 's' if fcnt > 1 else ''

{rda_python_dsquasar-3.0.21 → rda_python_dsquasar-3.0.22}/src/rda_python_dsquasar/dsquasar.usg

@@ -8,8 +8,8 @@
  file tarring and transferring is performed through the utility program dsarch.
 
  Usage: dsquasar [-b] [(-a | -t DatasetIDs)] [-c ChangeDays] [-(e|E)]     \
-                 [-(B|D)] [-A ActionBits] [-l (Y|N)] [-m ProcessCount]    \
-                 [-n] [-u] [-d [HostName] [TryCount]] [-h]
+                 [-(B|D)] [-L (G|O)] [-A ActionBits] [-l (Y|N)]           \
+                 [-m ProcessCount] [-n] [-u] [-d [HostName] [TryCount]] [-h]
 
    Either -a or -t must be given (usage is displayed otherwise).  dsquasar runs
    from the Quasar backup work directory; it changes there automatically.
@@ -24,6 +24,21 @@
           '%' is allowed and is expanded against the dataset table.  Ignored,
           with a warning, when -a is also given.
 
+   -L  (G|O)
+          Limit the run by dataset.locflag: O takes the datasets on the object
+          store, G takes everything else.  Applies to -a and to -t alike.  The
+          two classes back up at very different rates: an O dataset has every
+          file staged down from the object store and checksummed before it can
+          be tarred, while a G dataset is only stat'ed.  Running them in one job
+          lets the slow class eat the whole walltime, so give them separate
+          runs, e.g. one submit with -L G and another with -L O.  Defaults to
+          all datasets.
+
+          -L G and -L O together always cover every dataset, so a pair of runs
+          loses nothing.  The split is by dataset, though, and dataset.locflag
+          is only a summary of the per-file flags, so a dataset holding files of
+          both kinds is backed up entirely by whichever run claims it.
+
 
  BACKUP SCOPE
 

{rda_python_dsquasar-3.0.21 → rda_python_dsquasar-3.0.22/src/rda_python_dsquasar.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsquasar
-Version: 3.0.21
+Version: 3.0.22
 Summary: RDA Python package to backup and recover RDA data archives to and from GLOBUS Quasar backup server
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsquasar