rda-python-dsquasar 3.0.1__tar.gz → 3.0.6__tar.gz

rda_python_dsquasar-3.0.6/MANIFEST.in

@@ -0,0 +1 @@
+include src/rda_python_dsquasar/*.usg

{rda_python_dsquasar-3.0.1/src/rda_python_dsquasar.egg-info → rda_python_dsquasar-3.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsquasar
-Version: 3.0.1
+Version: 3.0.6
 Summary: RDA Python package to backup and recover RDA data archives to and from GLOBUS Quasar backup server
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsquasar

{rda_python_dsquasar-3.0.1 → rda_python_dsquasar-3.0.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_dsquasar"
-version = "3.0.1"
+version = "3.0.6"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]

{rda_python_dsquasar-3.0.1 → rda_python_dsquasar-3.0.6}/src/rda_python_dsquasar/dsquasar.py

@@ -12,7 +12,7 @@
 #             to gather Web and Saved bfiles, tar them into larger (> 2GB) bfiles, and
 #             back them up onto Quasar Backup Servers, at the Globus end points of
 #             NCAR GDEX Quasar and/or NCAR GDEX Quasar Drdata, for Backup and/or
-#             desaster recovery copies, respectively. The file tarring and backing up
+#             disaster recovery copies, respectively. The file tarring and backing up
 #             processes are completed in utility program dsarch.
 #
 #    Github : https://github.com/NCAR/rda-python-dsquasar.git
@@ -84,6 +84,25 @@ class DsQuasar(PgCMD, PgSplit):
       self.ONESIZE = 20*self.PGLOG['ONEGBS']  # 20GB, minimal file size to tar a single file 
       self.TFCOUNT = 100          # if file count is greater, use self.MINSIZE for tar file
       self.SUBLMTS = 2000         # file count limit for a sub-group
+      # tarring (TARACT) forks one child per tar file, so its parallel unit is the tar
+      # file: use a small per-process limit and a high cap to spread the work widely.
+      self.MPTLIMIT = 200         # tar file count per batch process for tarring
+      self.MPTMAX = 12            # maximum number of batch processes for tarring
+      # uploading (BCKACT) forks one child per BCKSIZE transfer batch (~BCKSIZE/TARSIZE
+      # tar files each), so far fewer parallel units than tar files: use a large
+      # per-process limit and a low cap to avoid over-reserving cpus.
+      self.MPBLIMIT = 900         # tar file count per batch process for uploading
+      self.MPBMAX = 4             # maximum number of batch processes for uploading
+      self.MAXRUNTIME = 23*3600   # 23 hours; stop before the 24-hour PBS walltime
+      self.ONEHOUR = 3600         # seconds; time headroom needed to finish before walltime
+      # a repeat submit normally blocks as a duplicate while the batch job is running. if
+      # that job is running but barely progressing, an extra worker is submitted instead
+      # (see pick_worker_slot). progress is measured as the fraction of the recorded
+      # dscheck work done, which is action independent because dcount counts GDEX files
+      # for -A 3 but tar files for -A 2/-A 4.
+      self.MAXWORKERS = 2         # default maximum concurrent workers per command (-W)
+      self.WORKGRACE = 6*3600     # 6 hours before a running job is judged on progress
+      self.MINWDONE = 0.01        # done fraction after WORKGRACE that counts as progressing
       self.PGBACK = {
          'workdir' : "{}/{}/quasar_backup".format(self.PGLOG['GDEXWORK'], self.PGLOG['COMMONUSER']),
          'mproc' : 1,
@@ -97,6 +116,10 @@ class DsQuasar(PgCMD, PgSplit):
          'maxcnt' : 10,
          'dolock' : 1,
          'doemail' : 0,
+         'starttime' : 0,   # wall-clock start of the run, for the PBS walltime guard
+         'tardone' : 0,     # tar files dispatched so far, for the finish-rate estimate
+         'maxworkers' : self.MAXWORKERS,   # -W, maximum concurrent workers per command
+         'worker' : 1,      # -w, this run's worker slot; >1 for an added extra worker
          'cmd'  : None
       }
       self.dsids = []
@@ -105,13 +128,15 @@ class DsQuasar(PgCMD, PgSplit):
       self.dstart = 0
 
    # read in command line parameters
-   def read_parameters(self):   
+   def read_parameters(self):
       self.PGLOG['LOGFILE'] = "dsquasar.log"   # set different log file
+      self.set_help_path(__file__)
       self.dssdb_dbname()
       option = None
       argv = sys.argv[1:]
       for arg in argv:
-         ms = re.match(r'^-(a|b|c|d|e|E|l|m|n|t|u|A|B|D)$', arg)
+         if re.match(r'^-(h|-help)$', arg, re.I): self.show_usage('dsquasar')
+         ms = re.match(r'^-(a|b|c|d|e|E|l|m|n|t|u|w|A|B|D|W)$', arg)
          if ms:
             arg = ms.group(1)
             if  arg == 'b':
@@ -119,7 +144,7 @@ class DsQuasar(PgCMD, PgSplit):
             elif arg in self.sopts:
                self.sopts[arg] = 1
                option = None
-            elif 'Acdlmt'.find(arg) > -1:
+            elif 'AcdlmtwW'.find(arg) > -1:
                option = arg
                if arg == 'd': self.bopts = []
             elif 'BD'.find(arg) > -1:
@@ -145,36 +170,28 @@ class DsQuasar(PgCMD, PgSplit):
                if not (arg == 'Y' or arg == 'N'): self.pglog(arg +": Lock Flag(-l) must be Y or N", self.LGWNEX)
                self.PGBACK['dolock'] = 1 if arg == 'Y' else 0
                option = None
+            elif option == 'W':
+               self.PGBACK['maxworkers'] = int(arg)
+               if self.PGBACK['maxworkers'] < 1: self.pglog(arg +": Maximum Workers(-W) must be > 0", self.LGWNEX)
+               option = None
+            elif option == 'w':
+               self.PGBACK['worker'] = int(arg)
+               if self.PGBACK['worker'] < 1: self.pglog(arg +": Worker Slot(-w) must be > 0", self.LGWNEX)
+               option = None
             else:
                self.add_to_dsids(arg)
          else:
             self.pglog(arg + ": Value without leading option", self.LGWNEX)
-      if not (self.sopts['a'] or self.dsids):
-         print("dsquasar [-b] [(-a|-t DatasetIDs)] [-c ChangeDays] [-(e|E)] [-(B|D)] [-A ActionBits] \\")
-         print("         [-l (Y|N)] [-m ProcessCount] [-n] [-u] [-d [HostName] [TryCount]")
-         print("  Option -a - gather files of all datasets for Quasar Backup, or")
-         print("  Option -t - specify dataset IDs to backup files, wildcard % is allowed")
-         print("  Option -b - turn the background mode on; no screen display")
-         print("  Option -c - change days to work with option -A 1 to rebackup changed files")
-         print("  Option -d - turn the delay mode on for running as a batch job")
-         print("  Option -e - send an email to specialist for statistics")
-         print("  Option -E - send an email to specialist for detail statistics")
-         print("  Option -B - Quasar Backup only for the provided datasets, or")
-         print("  Option -D - Quasar Backup&Drdata; default to both B & D")
-         print("  Option -A - Action bits, 1 - create dsarch input files; 2 - build tarfiles;")
-         print("              4 - Transfer Quasar backup files; 8 - Check Backup Files;")
-         print("              16 - Dump Backup Statistics; default to 3 for 1+2 actions,")
-         print("              and the valid values are 1,2,3,4,6,7,8,16")
-         print("  Option -l - flag to check&lock dataset before backup processes, defaults to Y")
-         print("  Option -m - number of concurrent processes, defaults to 1")
-         print("  Option -n - gather and show the available file counts with no backup actions")
-         print("  Option -u - to clean up backup locks for provided dataset IDs")
-         sys.exit(0)
+      if not (self.sopts['a'] or self.dsids): self.show_usage('dsquasar')
+      # an even worker slot walks the datasets from the other end so that it and the odd
+      # slot before it meet in the middle instead of fighting over the same dataset locks
+      if self.reverse_dsids(): self.dsids.reverse()
       self.PGBACK['cmd'] = "dsquasar {}".format(' '.join(argv))
 
    # do quasar backup process
    def start_actions(self):
       self.cmdlog(self.PGBACK['cmd'])
+      self.PGBACK['starttime'] = tm()
       if self.sopts['u']:
          if self.sopts['a']: self.pglog("-u: Dataset IDs must be provided to Unlock datasets", self.LOGWRN)
          self.unlock_datasets()
@@ -331,21 +348,148 @@ class DsQuasar(PgCMD, PgSplit):
          self.backup_dataset_tarfiles(dsfiles, 'B')
          self.backup_dataset_tarfiles(dsfiles, 'D')
 
+   # size up the number of batch processes to reserve, sizing tarring and uploading
+   # independently because their parallel work units differ. tarring's unit is the tar
+   # file (one child per tar): new infiles CINACT will create this run (estimated from
+   # the total bytes of GDEX files ready to back up, ~one tar per TARSIZE) plus existing
+   # status 'N' infile records to build (TARACT), scaled one process per MPTLIMIT tar
+   # files, capped at MPTMAX. uploading's unit is the BCKSIZE transfer batch (one child
+   # per ~BCKSIZE/TARSIZE tar files): existing status 'T' tar records to transfer
+   # (BCKACT), scaled one process per MPBLIMIT tar files, capped at MPBMAX. the phases
+   # run sequentially in one run and share a single reserved ncpus, so return the max of
+   # the two counts. create-infile only (-A 1) is not a batch action (see NBACTS) so it
+   # never reaches here and stays single process.
+   def batch_process_count(self, acts):
+      mproc = 1
+      tcnt = 0
+      if acts&self.CINACT:
+         sizes = [0]
+         if self.gather_dataset_files(None, False, sizes):
+            tcnt += max(1, (sizes[0] + self.TARSIZE - 1)//self.TARSIZE)
+      if acts&self.TARACT: tcnt += self.batch_tar_count('N')
+      if tcnt > self.MPTLIMIT:
+         tproc = (tcnt + self.MPTLIMIT - 1)//self.MPTLIMIT
+         if tproc > self.MPTMAX: tproc = self.MPTMAX
+         if tproc > mproc: mproc = tproc
+      if acts&self.BCKACT:
+         bcnt = self.batch_tar_count('T')
+         if bcnt > self.MPBLIMIT:
+            bproc = (bcnt + self.MPBLIMIT - 1)//self.MPBLIMIT
+            if bproc > self.MPBMAX: bproc = self.MPBMAX
+            if bproc > mproc: mproc = bproc
+      return mproc
+
+   # count the bfile tar records of a given status honoring backflag/dataset scope
+   def batch_tar_count(self, status):
+      bcnd = "status = '{}'".format(status)
+      if self.PGBACK['backflag']: bcnd += " AND type = '{}'".format(self.PGBACK['backflag'])
+      if self.dsids:
+         tcnt = 0
+         for dsid in self.dsids:
+            tcnt += self.pgget('bfile', '', "dsid = '{}' AND {}".format(dsid, bcnd), self.LGWNEX)
+         return tcnt
+      return self.pgget('bfile', '', bcnd, self.LGWNEX)
+
+   # look up the dscheck record registered for a worker's argv, using the same lookup key
+   # init_dscheck builds
+   def lookup_worker_dscheck(self, argv):
+      dargv = self.argv_to_string(argv, 0, "Process in Delayed Mode")
+      dargx = None
+      if len(dargv) > 100:
+         dargx = dargv[100:]
+         dargv = dargv[0:100]
+      return self.get_dscheck("dsquasar", dargv, self.PGBACK['workdir'], self.PGLOG['CURUID'], dargx, self.LOGWRN)
+
+   # a running batch job counts as stalled once it has run longer than WORKGRACE and has
+   # still completed no more than MINWDONE of its recorded work; anything further along
+   # than that is progressing and is left alone. the done fraction dcount/fcount is used
+   # instead of a raw count because dcount counts GDEX files for -A 3 but tar files for
+   # -A 2/-A 4. a job that is not locked, not started yet, or whose process is gone is not
+   # stalled: init_dscheck already restarts a dead one.
+   def worker_progress_stalled(self, pgrec):
+      if not (pgrec['pid'] and pgrec['stttime']): return False
+      if self.check_host_pid(pgrec['lockhost'], pgrec['pid']) <= 0: return False
+      if (tm() - pgrec['stttime']) < self.WORKGRACE: return False
+      done = pgrec['dcount']/pgrec['fcount'] if pgrec['fcount'] else 0
+      return done <= self.MINWDONE
+
+   # whether more than one worker may run the current action. two workers stay off each
+   # other's files only through dataset locking, so the action must lock what it works on:
+   # creating input files and building tar files do (backup_dataset_files and
+   # backup_dataset_infiles lock each dsid), but transferring (BCKACT) walks the status 'T'
+   # records by bid without locking, so two workers would transfer the same tar files twice.
+   def extra_workers_allowed(self):
+      return bool(self.PGBACK['dolock'] and not self.PGBACK['action']&self.BCKACT)
+
+   # pick the worker slot to submit into. slot 1 uses the invocation argv unchanged so a
+   # repeat submit still blocks as a duplicate. when a slot is held by a running job that
+   # is stalled, the next slot is tried instead, and appending -w N to the argv gives that
+   # extra worker its own dscheck record and PBS job. returns the slot and the dscheck
+   # record already registered for it, if any.
+   def pick_worker_slot(self):
+      if self.PGBACK['worker'] > 1:   # an explicit -w N forces that slot
+         return (self.PGBACK['worker'], self.lookup_worker_dscheck(sys.argv[1:]))
+      pgrec = self.lookup_worker_dscheck(sys.argv[1:])
+      if not pgrec or not self.worker_progress_stalled(pgrec): return (1, pgrec)
+      # worker 1 is running but stalled: submit an extra worker into the first slot with no
+      # dscheck record yet. a slot that is already registered is left alone, so an extra
+      # worker still waiting to be started by the dscheck daemon is never taken over and
+      # run here instead, outside PBS.
+      maxw = self.PGBACK['maxworkers'] if self.extra_workers_allowed() else 1
+      for slot in range(2, maxw + 1):
+         if not self.lookup_worker_dscheck(sys.argv[1:] + ['-w', str(slot)]):
+            self.pglog("Worker 1 is running with {} of {} done: adding worker {}".format(
+                       pgrec['dcount'], pgrec['fcount'], slot), self.LOGWRN)
+            return (slot, None)
+      return (1, pgrec)   # no free slot; block on worker 1 as usual
+
    # start none daemon and intialize dscheck counts
    def start_dsquasar_none_daemon(self, act, fcnt = 0):
       acts = self.PGBACK['action']
+      cact = ('A' if acts < 10 else '') + str(acts)
       if self.bopts != None:
          dsid = self.dsids[0] if len(self.dsids) == 1 else ''
          if not re.match(r'^[a-z]\d{6}$', dsid): dsid = ''
-         cact = ('A' if acts < 10 else '') + str(acts)
-         if act == self.STTACT or acts&self.NBACTS != acts:
-            self.set_one_boption('qoptions', '-l walltime=24:00:00', 1)
+         # only the initial delayed submit (not yet under PBS) sets qoptions for the batch
+         # job the dscheck daemon will start; size up multi-processing from the detail file
+         # count (unless an explicit -m N was given) and reserve matching PBS cpus. a
+         # command-line or the PBS batch run does not set qoptions.
+         if self.PGLOG['CURBID'] < 1 and (act == self.STTACT or acts&self.NBACTS != acts):
+            # pick the worker slot first; slot 1 keeps the invocation argv so a repeat
+            # submit still blocks as a duplicate, while a stalled run hands the next slot
+            # its own -w N argv. only size up multi-processing - which checks the backup
+            # files - when the chosen slot has no dscheck record on file yet, so a
+            # duplicate submit blocks without checking the backup files.
+            (slot, pgrec) = self.pick_worker_slot()
+            if slot > 1:
+               self.PGBACK['worker'] = slot
+               sys.argv.extend(['-w', str(slot)])
+               self.PGBACK['cmd'] += " -w {}".format(slot)
+            if not pgrec:
+               qoptions = '-l walltime=24:00:00'
+               if acts&self.NBACTS != acts:
+                  if self.PGBACK['mproc'] < 2: self.PGBACK['mproc'] = self.batch_process_count(acts)
+                  if self.PGBACK['mproc'] > 1:
+                     qoptions += ",select=1:ncpus={0}:mem={0}gb".format(self.PGBACK['mproc'])
+               self.set_one_boption('qoptions', qoptions, 1)
          self.init_dscheck(0, '', "dsquasar", dsid, cact, self.PGBACK['workdir'],
                             self.PGLOG['CURUID'], self.bopts, self.LOGWRN)
-         if self.PGBACK['mproc'] > 1: self.PGBACK['mproc'] = 1
+         # on the PBS batch run, take the process count from the reserved ncpus in qoptions
+         # instead of recomputing it
+         if self.PGLOG['DSCHECK'] and self.PGBACK['mproc'] < 2 and acts&self.NBACTS != acts:
+            pgrec = self.pgget('dscheck', 'qoptions',
+                               "cindex = {}".format(self.PGLOG['DSCHECK']['cindex']), self.LGEREX)
+            if pgrec and pgrec['qoptions']:
+               ms = re.search(r'ncpus=(\d+)', pgrec['qoptions'])
+               if ms: self.PGBACK['mproc'] = int(ms.group(1))
+         if self.PGBACK['mproc'] > 1 and acts&self.NBACTS == acts: self.PGBACK['mproc'] = 1
       elif self.PGBACK['mproc'] > 1 and acts&self.NBACTS == acts:
          self.PGBACK['mproc'] = 1
-      self.start_none_daemon('dsquasar', '', self.PGLOG['CURUID'], self.PGBACK['mproc'], 60, 1)
+      # tag the daemon string with the action, and with the worker slot for an extra worker,
+      # so that concurrent workers can be told apart in the log
+      dact = cact
+      if self.PGBACK['worker'] > 1: dact += "-w{}".format(self.PGBACK['worker'])
+      self.start_none_daemon('dsquasar', dact, self.PGLOG['CURUID'], self.PGBACK['mproc'], 60, 1)
       if self.PGLOG['DSCHECK']:
          if act == self.STTACT:
             fcnt = self.gather_dataset_bckfiles(None, False)
@@ -409,13 +553,17 @@ class DsQuasar(PgCMD, PgSplit):
       dmsg = list(bfiles)[0] if dcnt == 1 else "{} datasets".format(dcnt)
       self.pglog("{}: {} for {} ...".format(amsg, bmsg, dmsg), self.WARNLG)
       qinfo = {'backflag' : backflag, 'bid' : self.current_bid(), 'dsids' : [], 'fcnt' : 0,
-               'size' : 0, 'infiles' : [], 'instr' : '', 'qdsids' : [],
+               'size' : 0, 'infiles' : [], 'instr' : '', 'qdsids' : [], 'abids' : [],
                'dslocks' : [], 'qfcnt' : 0, 'qsize' : 0, 'qcnt' : 0}
       for dsid in bfiles:
          if len(qinfo['dsids']) == self.DSCNT: self.process_one_backup_file(qinfo, True, False)
          if self.PGBACK['dolock'] and dsid not in qinfo['dslocks']:
             if self.lock_dataset(dsid, 1, self.LOGERR) < 1: continue
             qinfo['dslocks'].append(dsid)
+            if not self.regather_dataset_files(bfiles, dsid, backflag):
+               self.lock_dataset(dsid, 0, self.LGEREX)
+               qinfo['dslocks'].remove(dsid)
+               continue
          fcnt = bfiles[dsid]['scount']
          if fcnt > 0:
             self.process_backup_files(qinfo, dsid, fcnt, bfiles[dsid]['srecs'], 'S')
@@ -427,6 +575,9 @@ class DsQuasar(PgCMD, PgSplit):
                self.lock_dataset(dsid, 0, self.LGEREX)
                qinfo['dslocks'].remove(dsid)
             qinfo['dsids'] = []
+      if self.PGBACK['mproc'] > 1:
+         self.check_child(None, 0, self.LOGWRN, 1)   # wait all child processes done
+         self.confirm_quasar_counts(qinfo, qinfo['abids'], "status = 'T'")   # recount confirmed backups from RDADB
       if qinfo['dslocks']:
          for dsid in qinfo['dslocks']:
             self.lock_dataset(dsid, 0, self.LGEREX)  # unlock the locked datasets
@@ -453,6 +604,44 @@ class DsQuasar(PgCMD, PgSplit):
          msg = "{}: {}({}) GDEX {}file{} of {} for next {}".format(amsg, fcnt, ssize, cmsg, s, dmsg, bmsg)
          self.pglog(self.INDENT + msg, self.LOGACT)
 
+   # the file list was gathered before this dataset was locked, so it can be hours old and
+   # name files another worker has backed up since. re-gather it now that the dataset is
+   # locked and no other worker can add backup files for it, and replace the stale list.
+   # returns the file count left to back up for the given backup flag.
+   def regather_dataset_files(self, bfiles, dsid, backflag):
+      pgrec = self.pgget("dataset", "backflag", "dsid = '{}'".format(dsid), self.LGWNEX)
+      if not pgrec: return 0
+      dsfiles = {'B' : {}, 'D' : {}}
+      self.get_dataset_files(dsid, dsfiles, pgrec['backflag'])
+      if dsid not in dsfiles[backflag]:
+         self.pglog("{}: no {} file left to back up, skip".format(dsid, self.BACKMSG[backflag]), self.DTLACT)
+         return 0
+      bfiles[dsid] = dsfiles[backflag][dsid]
+      return bfiles[dsid]['scount'] + bfiles[dsid]['wcount']
+
+   # lock the other datasets whose files share one tar file, so that no other worker gathers
+   # or tars their files while this tar file is being claimed. datasets already locked by this
+   # run, including the primary one, are left alone. returns the datasets locked here, to be
+   # unlocked once the tar file is dispatched, or None if one of them is held by another
+   # worker, in which case nothing new is left locked and the tar file must wait for a later run
+   def lock_backup_datasets(self, qinfo, dsids):
+      dslocks = []
+      if not self.PGBACK['dolock']: return dslocks
+      for dsid in dsids:
+         if dsid in qinfo['dslocks']: continue
+         if self.lock_dataset(dsid, 1, self.LOGERR) < 1:
+            self.unlock_backup_datasets(qinfo, dslocks)
+            return None
+         qinfo['dslocks'].append(dsid)
+         dslocks.append(dsid)
+      return dslocks
+
+   # unlock the given datasets and drop them from the lock list tracked for cleanup on quit
+   def unlock_backup_datasets(self, qinfo, dslocks):
+      for dsid in dslocks:
+         self.lock_dataset(dsid, 0, self.LOGERR)
+         if dsid in qinfo['dslocks']: qinfo['dslocks'].remove(dsid)
+
    # get the current bid for adding bfile
    def current_bid(self):
       pgrec = self.pgget("bfile", "max(bid) mid", '', self.LGEREX)
@@ -475,16 +664,36 @@ class DsQuasar(PgCMD, PgSplit):
       bmsg = self.BACKMSG[backflag]
       s = 's' if bcnt > 1 else ''
       self.pglog("{}: {} {} file{}...".format(amsg, bcnt, bmsg, s), self.WARNLG)
-      qinfo = {'backflag' : backflag, 'bid' : 0, 'dsids' : [], 'fcnt' : 0, 'size' : 0,
-               'infiles' : [], 'instr' : '', 'qdsids' : [], 'qfcnt' : 0, 'qsize' : 0, 'qcnt' : 0}
+      qinfo = {'backflag' : backflag, 'bid' : 0, 'dsids' : [], 'fcnt' : 0, 'size' : 0, 'abids' : [],
+               'infiles' : [], 'instr' : '', 'qdsids' : [], 'dslocks' : [], 'qfcnt' : 0,
+               'qsize' : 0, 'qcnt' : 0}
       for dsid in bfiles:
-         if self.PGBACK['dolock'] and self.lock_dataset(dsid, 1, self.LOGERR) < 1: continue
+         if self.PGBACK['dolock']:
+            if self.lock_dataset(dsid, 1, self.LOGERR) < 1: continue
+            qinfo['dslocks'].append(dsid)
          for bid in bfiles[dsid]:
+            # the status 'N' records were gathered before this dataset was locked, so they
+            # can be hours old; confirm each one is still untarred before spending a tar on
+            # it. holding the dataset lock makes this check race free against a second
+            # worker, which cannot hold the same dsid at the same time.
+            if not self.pgget('bfile', '', "bid = {} AND status = 'N'".format(bid), self.LGEREX):
+               self.pglog("{}-{}: no longer waiting to be tarred, skip".format(dsid, bid), self.DTLACT)
+               continue
             qinfo['bid'] = bid
             binfo = bfiles[dsid][bid]
             for bkey in binfo: qinfo[bkey] = binfo[bkey]
+            # a tar file can hold files of multiple datasets; lock the other ones too before
+            # tarring, and release them again as soon as the tar file is dispatched
+            dslocks = self.lock_backup_datasets(qinfo, binfo['dsids'])
+            if dslocks is None:
+               self.pglog("{}-{}: another dataset of the tar file is locked, skip".format(dsid, bid), self.DTLACT)
+               continue
             self.process_one_backup_file(qinfo, False)
-         if self.PGBACK['dolock']: self.lock_dataset(dsid, 0, self.LOGERR)
+            self.unlock_backup_datasets(qinfo, dslocks)
+         if self.PGBACK['dolock']: self.unlock_backup_datasets(qinfo, [dsid])
+      if self.PGBACK['mproc'] > 1:
+         self.check_child(None, 0, self.LOGWRN, 1)   # wait all child processes done
+         self.confirm_quasar_counts(qinfo, qinfo['abids'], "status = 'T'")   # recount confirmed backups from RDADB
       qcnt = qinfo['qcnt']
       if qcnt > 0:
          self.PGBACK['bckcnt'] += qcnt
@@ -522,6 +731,9 @@ class DsQuasar(PgCMD, PgSplit):
          if qinfo['size'] < self.BCKSIZE: continue
          self.transfer_quasar_tarfiles(qinfo)
       if qinfo['size'] > 0: self.transfer_quasar_tarfiles(qinfo)
+      if self.PGBACK['mproc'] > 1:
+         self.check_child(None, 0, self.LOGWRN, 1)   # wait all child processes done
+         self.confirm_quasar_counts(qinfo, list(bfiles), "status = 'A'")   # recount confirmed backups from RDADB
       qcnt = qinfo['qcnt']
       if qcnt > 0:
          self.PGBACK['bckcnt'] += qcnt
@@ -562,11 +774,79 @@ class DsQuasar(PgCMD, PgSplit):
       if self.PGBACK['pstep']: self.record_dscheck_status("F")
       self.pgexit(0)
 
+   # guard the -A 3 (Create Input&Tar, build status 'N' infiles) and -A 4 (Transfer,
+   # send status 'T' tars) PBS batch jobs against the 24-hour walltime: once past
+   # MAXRUNTIME, if the tar files still to process cannot finish within the last hour at
+   # the rate achieved so far, send a progress email and stop cleanly so the report is
+   # not lost to a PBS timeout; the remaining records are left for the next scheduled run
+   # to resume. a no-op for command-line runs, other actions, or before the cutoff.
+   def check_batch_deadline(self, qinfo):
+      act = self.PGBACK['action']
+      if self.PGLOG['CURBID'] < 1 or act not in (self.CTACTS, self.BCKACT): return
+      elapsed = tm() - self.PGBACK['starttime']
+      if elapsed < self.MAXRUNTIME: return
+      status, work = ('N', 'build') if act == self.CTACTS else ('T', 'transfer')
+      remaining = self.batch_tar_count(status)
+      if remaining < 1: return
+      done = self.PGBACK['tardone']
+      need = remaining*elapsed/done if done > 0 else elapsed
+      if need <= self.ONEHOUR: return   # enough time left to finish before the walltime
+      if self.PGBACK['mproc'] > 1: self.check_child(None, 0, self.LOGWRN, 1)   # wait all children
+      if qinfo and qinfo.get('dslocks'):
+         for dsid in qinfo['dslocks']: self.lock_dataset(dsid, 0, self.LGEREX)
+      etime = self.seconds_to_string_time(int(elapsed))
+      amsg = self.ACTMSG[act]
+      msg = "{}: Stopped after {} with {} tar file(s) still to {} - not enough time to finish before the 24-hour PBS walltime; the next scheduled run will resume".format(amsg, etime, remaining, work)
+      self.pglog(self.INDENT + msg, self.LOGACT)
+      if self.PGBACK['doemail']:
+         bmsg = self.BACKMSG[self.PGBACK['backflag']] if self.PGBACK['backflag'] else 'backup'
+         self.set_email("{}: {} - stopped early before the PBS walltime with {} {} tar file(s) remaining!".format(self.PGBACK['cmd'], amsg, remaining, bmsg), self.EMLTOP)
+         title = "dsquasar: {} stopped early ({} remaining)".format(amsg, remaining)
+         if self.PGBACK['errcnt']: title += " Error({})".format(self.PGBACK['errcnt'])
+         if self.PGLOG['DSCHECK']:
+            self.build_customized_email("dscheck", "einfo", "cindex = {}".format(self.PGLOG['DSCHECK']['cindex']), title, self.LOGWRN)
+         else:
+            self.pglog(title, self.LOGWRN|self.SNDEML)
+      if self.PGBACK['pstep']: self.record_dscheck_status("D")
+      self.pgexit(0)
+
+   # recompute the confirmed backup counts from RDADB after all child processes
+   # finished, so a multi-process summary reflects succeeded (not just started) work
+   def confirm_quasar_counts(self, qinfo, bids, dcnd):
+      qinfo['qcnt'] = qinfo['qfcnt'] = qinfo['qsize'] = 0
+      qinfo['qdsids'] = []
+      if not bids: return
+      cnd = "{} AND bid IN ({})".format(dcnd, ','.join(str(bid) for bid in bids))
+      pgrecs = self.pgmget('bfile', "dsid, dsids, data_size size, (scount + wcount) fcnt", cnd, self.LGEREX)
+      bcnt = len(pgrecs['dsid']) if pgrecs else 0
+      for i in range(bcnt):
+         qinfo['qcnt'] += 1
+         qinfo['qfcnt'] += pgrecs['fcnt'][i]
+         qinfo['qsize'] += pgrecs['size'][i]
+         dsid = pgrecs['dsid'][i]
+         if dsid not in qinfo['qdsids']: qinfo['qdsids'].append(dsid)
+         if pgrecs['dsids'][i]:
+            for did in pgrecs['dsids'][i].split(','):
+               if did and did not in qinfo['qdsids']: qinfo['qdsids'].append(did)
+
+   # dsarch reports "<dsid>-<file>: <type> file backed up to /<dsid>/<bfile> by <date>" for
+   # each file to tar that is already backed up. our own record is the duplicate to drop only
+   # when those files point at some other bfile; a report naming our own bfile means this
+   # record is the one already backed up, so it must be kept.
+   def duplicate_backup_file(self, dsid, qfile):
+      ours = "/{}/{}".format(dsid, qfile)
+      isdup = False
+      for ms in re.finditer(r'file backed up to (\S+) by ', self.PGLOG['SYSERR']):
+         if ms.group(1) == ours: return False
+         isdup = True
+      return isdup
+
    # backup one Quasar Backup or Backup&Drdata from one or multiple inputs and,
    # reset the quasar backup dict
    def process_one_backup_file(self, qinfo, addback, keepid = False):
       ccnt = self.check_child(None, 0, self.LOGWRN, -1) if self.PGBACK['mproc'] > 1 else 0
       if self.PGSIG['QUIT']: self.quit_dsquasar(qinfo)
+      self.check_batch_deadline(qinfo)
       dsids = qinfo['dsids']
       dcnt = len(dsids)
       if dcnt == 0: return
@@ -580,6 +860,7 @@ class DsQuasar(PgCMD, PgSplit):
       else:
          bid = qinfo['bid']
          qfile = qinfo['bfile']
+      if 'abids' in qinfo: qinfo['abids'].append(bid)   # track dispatched bids for confirmed recount
       stat = 1
       if self.PGBACK['action'] > self.CINACT:
          s = 's' if fcnt > 1 else ''
@@ -599,9 +880,9 @@ class DsQuasar(PgCMD, PgSplit):
                stat = self.pgsystem(cmd, self.ERRACT, 325)   # 256 + 64 + 4 + 1
                if stat:
                   for infile in qinfo['infiles']: self.delete_local_file(infile)
-               elif re.search(r'file backed up to', self.PGLOG['SYSERR']):
+               elif self.duplicate_backup_file(dsid, qfile):
                   if self.pgdel('bfile', f"bid = {bid}"):
-                     self.pglog(f"{dsid}-{qfile}: backup tarfile deleted for duplicattion", self.DTLACT)
+                     self.pglog(f"{dsid}-{qfile}: backup tarfile deleted for duplication", self.DTLACT)
                   for infile in qinfo['infiles']: self.delete_local_file(infile)
                sys.exit(0)  # stop child process
             else:
@@ -612,9 +893,9 @@ class DsQuasar(PgCMD, PgSplit):
             stat = self.pgsystem(cmd, self.ERRACT, 325)   # 256 + 64 + 4 + 1
             if stat:
                for infile in qinfo['infiles']: self.delete_local_file(infile)
-            elif re.search(r'file backed up to', self.PGLOG['SYSERR']):
+            elif self.duplicate_backup_file(dsid, qfile):
                if self.pgdel('bfile', f"bid = {bid}"):
-                  self.pglog(f"{dsid}-{qfile}: backup tarfile deleted for duplicattion", self.DTLACT)
+                  self.pglog(f"{dsid}-{qfile}: backup tarfile deleted for duplication", self.DTLACT)
                for infile in qinfo['infiles']: self.delete_local_file(infile)
             else:
                self.PGBACK['errcnt'] += 1
@@ -623,12 +904,13 @@ class DsQuasar(PgCMD, PgSplit):
       if stat:
          # reset qinfo after quasar backup
          qinfo['qcnt'] += 1
+         self.PGBACK['tardone'] += 1   # cumulative dispatched tars for the walltime finish-rate
          qinfo['qfcnt'] += fcnt
          qinfo['qsize'] += fsize
          for dsid in dsids:
             if dsid not in qinfo['qdsids']: qinfo['qdsids'].append(dsid)
       if addback:
-         lastdsid = dsids.pop() if keepid else None         
+         lastdsid = dsids.pop() if keepid else None
          for dsid in dsids:  # unlock all datasets but the last one
             if dsid in qinfo['dslocks']:
                self.lock_dataset(dsid, 0, self.LGEREX)
@@ -645,6 +927,7 @@ class DsQuasar(PgCMD, PgSplit):
    def transfer_quasar_tarfiles(self, qinfo):
       ccnt = self.check_child(None, 0, self.LOGWRN, -1) if self.PGBACK['mproc'] > 1 else 0
       if self.PGSIG['QUIT']: self.quit_dsquasar(qinfo)
+      self.check_batch_deadline(qinfo)
       # prepare for backup one tar file
       dsids = qinfo['dsids']
       bids = qinfo['bids']
@@ -675,10 +958,10 @@ class DsQuasar(PgCMD, PgSplit):
       dstat = bstat = -1
       if ccnt == 0 or self.PGSIG['PPID'] > 1:
          if bflg == 'D':
-            dstat = self.quasar_multiple_trasnfer(tofiles, fromfiles, 'gdex-quasar-drdata', 'gdex-glade', self.ERRACT)
+            dstat = self.quasar_multiple_transfer(tofiles, fromfiles, 'gdex-quasar-drdata', 'gdex-glade', self.ERRACT)
             if not dstat: self.pglog("Error Quaser Drdata for " + fmsg, self.ERRACT|self.LOGERR)
-         bstat = self.quasar_multiple_trasnfer(tofiles, fromfiles, 'gdex-quasar', 'gdex-glade', self.ERRACT)
-         if not dstat: self.pglog("Error Quaser Backup for " + fmsg, self.ERRACT|self.LOGERR)
+         bstat = self.quasar_multiple_transfer(tofiles, fromfiles, 'gdex-quasar', 'gdex-glade', self.ERRACT)
+         if not bstat: self.pglog("Error Quaser Backup for " + fmsg, self.ERRACT|self.LOGERR)
          if dstat == self.FINISH: dstat = self.check_globus_finished(tofiles[0], 'gdex-quasar-drdata', self.ERRACT|self.NOWAIT)
          if bstat == self.FINISH: bstat = self.check_globus_finished(tofiles[0], 'gdex-quasar', self.ERRACT|self.NOWAIT)
          if dstat and bstat:
@@ -698,6 +981,7 @@ class DsQuasar(PgCMD, PgSplit):
       if dstat and bstat:
          # reset qinfo after quasar backup
          qinfo['qcnt'] += bcnt
+         self.PGBACK['tardone'] += bcnt   # cumulative transferred tars for the walltime finish-rate
          qinfo['qfcnt'] += fcnt
          qinfo['qsize'] += fsize
          for dsid in dsids:
@@ -777,7 +1061,7 @@ class DsQuasar(PgCMD, PgSplit):
       return {'wcount' : 0, 'wrecs' : [], 'scount' : 0, 'srecs' : []}
 
    # get all files need to be backed up for a given dataset ID
-   def get_dataset_files(self, dsid, dsfiles, backflag, logact = 0):
+   def get_dataset_files(self, dsid, dsfiles, backflag, logact = 0, sizes = None):
       dcnd = "dsid = '{}'".format(dsid)
       fopt = self.get_backup_options(dsid, dcnd)
       if not fopt:
@@ -817,6 +1101,7 @@ class DsQuasar(PgCMD, PgSplit):
                self.pglog("{}-G{}: Found {} Files ({})".format(dsid, gidx, rcnt, rmsg), self.LOGWRN)
    
       fcnt = cnts['B'][0] + cnts['D'][0]
+      if sizes is not None: sizes[0] += cnts['B'][1] + cnts['D'][1]
       if dsfiles:
          cmsg = 'Changed ' if self.PGBACK['chgdays'] else ''
          for bflg in dsfiles:
@@ -836,7 +1121,6 @@ class DsQuasar(PgCMD, PgSplit):
 
    # get web/saved files to backup for a given group condition and file option
    def get_group_files(self, dsid, gcnd, dcnd, bfiles, fopt, infiles, cnts):
-      rcnt = 0
       if self.PGBACK['chgdays'] > 0:
          return self.get_group_changed_files(dsid, gcnd, dcnd, bfiles, fopt, infiles, cnts)
       else:
@@ -982,28 +1266,37 @@ class DsQuasar(PgCMD, PgSplit):
    def get_backup_options(self, dsid, dcnd):
       fopt = 0
       bcnd = "= 0" if self.PGBACK['chgdays'] < 1 else "> 0"
-   #   if self.pgget('sfile', 'sid', "{} AND bid {}".format(dcnd, bcnd), self.LGWNEX): fopt |= self.SOPT
       if self.pgget_wfile(dsid, 'wid', "bid " + bcnd, self.LGWNEX): fopt |= self.WOPT
       return fopt
 
+   # dataset ordering for gathering work; even worker slots start from the high end (d999999)
+   # and odd ones from the low end, so that a pair of workers meets in the middle instead of
+   # fighting over the same dataset locks
+   def dsid_order(self):
+      return " ORDER BY dsid DESC" if self.reverse_dsids() else " ORDER BY dsid"
+
+   # whether this worker slot walks the datasets from the high end
+   def reverse_dsids(self):
+      return self.PGBACK['worker']%2 == 0
+
    # gather all available dataset ids to backup data files
-   def gather_dataset_files(self, dsfiles, unlock = True):
+   def gather_dataset_files(self, dsfiles, unlock = True, sizes = None):
       fcnt = 0
+      # warn per named dataset with no files; stay silent when scanning all datasets
+      logact = self.LOGWRN if self.dsids else 0
       if self.dsids:
+         pgrecs = []
          for dsid in self.dsids:
-            dcnd = "dsid = '{}'".format(dsid)
-            pgrec = self.pgget("dataset", "dsid, backflag, pid", dcnd, self.LGWNEX)
-            if pgrec:
-               if unlock and pgrec['pid'] and self.lock_dataset(dsid, 0, self.LOGACT) < 1: continue
-               fcnt += self.get_dataset_files(dsid, dsfiles, pgrec['backflag'], self.LOGWRN)
+            pgrec = self.pgget("dataset", "dsid, backflag, pid", "dsid = '{}'".format(dsid), self.LGWNEX)
+            if pgrec: pgrecs.append(pgrec)
       else:
-         dcnd = "ORDER BY dsid"
-         pgrecs = self.pgmget("dataset", "dsid, backflag, pid", dcnd, self.LGWNEX)
-         dcnt = len(pgrecs['dsid']) if pgrecs else 0
-         for i in range(dcnt):
-            dsid = pgrecs['dsid'][i]
-            if unlock and pgrecs['pid'][i] and self.lock_dataset(dsid, 0, self.LOGACT) < 1: continue
-            fcnt += self.get_dataset_files(dsid, dsfiles, pgrecs['backflag'][i])
+         mrecs = self.pgmget("dataset", "dsid, backflag, pid", self.dsid_order().strip(), self.LGWNEX)
+         dcnt = len(mrecs['dsid']) if mrecs else 0
+         pgrecs = [self.onerecord(mrecs, i) for i in range(dcnt)]
+      for pgrec in pgrecs:
+         dsid = pgrec['dsid']
+         if unlock and pgrec['pid'] and self.lock_dataset(dsid, 0, self.LOGACT) < 1: continue
+         fcnt += self.get_dataset_files(dsid, dsfiles, pgrec['backflag'], logact, sizes)
       if dsfiles:
          s = 's' if fcnt > 1 else ''
          bmsg = self.BACKMSG[self.PGBACK['backflag']] if self.PGBACK['backflag'] else 'backup' 
@@ -1013,28 +1306,21 @@ class DsQuasar(PgCMD, PgSplit):
 
    # gather all available backup file records with status N to backup
    def gather_dataset_infiles(self, dsfiles):
-      icnt = fcnt = cnt = 0
+      icnt = fcnt = 0
       bcnd = "status = 'N'"
-      ocnd = " ORDER BY dsid"
       flds = "bid, dsid, bfile, type, data_size size, (scount + wcount) fcnt, dsids, note"
       if self.PGBACK['backflag']: bcnd += " AND type = '{}'".format(self.PGBACK['backflag'])
       if self.dsids:
-         for dsid in self.dsids:
-            dcnd = "dsid = '{}' AND {}".format(dsid, bcnd)
-            pgrecs = self.pgmget("bfile", flds, dcnd, self.LGWNEX)
-            bcnt = len(pgrecs['bid']) if pgrecs else 0
-            for i in range(bcnt):
-               cnt = self.get_backup_infile(dsfiles, self.onerecord(pgrecs, i))
-               if cnt:
-                  icnt +=1
-                  fcnt += cnt
+         cnds = ["dsid = '{}' AND {}".format(dsid, bcnd) for dsid in self.dsids]
       else:
-         pgrecs = self.pgmget("bfile", flds, bcnd+ocnd, self.LGWNEX)
+         cnds = [bcnd + self.dsid_order()]
+      for dcnd in cnds:
+         pgrecs = self.pgmget("bfile", flds, dcnd, self.LGWNEX)
          bcnt = len(pgrecs['bid']) if pgrecs else 0
          for i in range(bcnt):
             cnt = self.get_backup_infile(dsfiles, self.onerecord(pgrecs, i))
             if cnt:
-               icnt +=1
+               icnt += 1
                fcnt += cnt
       if dsfiles:
          s = 's' if icnt > 1 else ''
@@ -1045,28 +1331,21 @@ class DsQuasar(PgCMD, PgSplit):
    # gather all available backup file records with status T to backup
    #
    def gather_dataset_tarfiles(self, dsfiles):
-      tcnt = fcnt = cnt = 0
+      tcnt = fcnt = 0
       bcnd = "status = 'T'"
-      ocnd = " ORDER BY dsid"
       flds = "bid, dsid, bfile, type, data_size size, (scount + wcount) fcnt, note"
       if self.PGBACK['backflag']: bcnd += " AND type = '{}'".format(self.PGBACK['backflag'])
       if self.dsids:
-         for dsid in self.dsids:
-            dcnd = "dsid = '{}' AND {}".format(dsid, bcnd)
-            pgrecs = self.pgmget("bfile", flds, dcnd, self.LGWNEX)
-            bcnt = len(pgrecs['bid']) if pgrecs else 0
-            for i in range(bcnt):
-               cnt = self.get_backup_tarfile(dsfiles, self.onerecord(pgrecs, i))
-               if cnt:
-                  tcnt +=1
-                  fcnt += cnt
+         cnds = ["dsid = '{}' AND {}".format(dsid, bcnd) for dsid in self.dsids]
       else:
-         pgrecs = self.pgmget("bfile", flds, bcnd+ocnd, self.LGWNEX)
+         cnds = [bcnd + self.dsid_order()]
+      for dcnd in cnds:
+         pgrecs = self.pgmget("bfile", flds, dcnd, self.LGWNEX)
          bcnt = len(pgrecs['bid']) if pgrecs else 0
          for i in range(bcnt):
             cnt = self.get_backup_tarfile(dsfiles, self.onerecord(pgrecs, i))
             if cnt:
-               tcnt +=1
+               tcnt += 1
                fcnt += cnt
       s = 's' if tcnt > 1 else ''
       bmsg = self.BACKMSG[self.PGBACK['backflag']] if self.PGBACK['backflag'] else 'backup' 
@@ -1092,7 +1371,6 @@ class DsQuasar(PgCMD, PgSplit):
       if self.PGBACK['action']&self.CINACT: ret = fcnt
       if not dsfiles: return ret
       dsids = [dsid]
-   #   if not (pgrec['note'] or self.rebuild_file_note(pgrec, dsids)): return 0
       if not pgrec['note']:
          return self.pglog(bfile + ": Miss note for N backup file", self.ERRACT)
       infiles = []
@@ -1147,7 +1425,6 @@ class DsQuasar(PgCMD, PgSplit):
       if self.PGBACK['action']&self.CINACT: ret = fcnt
       if not dsfiles: return ret
       dsids = [dsid]
-   #   if not (pgrec['note'] or self.rebuild_file_note(pgrec, dsids)): return 0
       if not pgrec['note']:
          return self.pglog(bfile + ": Miss note for T backup file", self.ERRACT)
       ftype = None
@@ -1347,6 +1624,9 @@ class DsQuasar(PgCMD, PgSplit):
          binfo = bfiles[bid]
          for bkey in binfo: qinfo[bkey] = binfo[bkey]
          self.process_one_quasar_pathfile(qinfo)
+      if self.PGBACK['mproc'] > 1:
+         self.check_child(None, 0, self.LOGWRN, 1)   # wait all child processes done
+         self.confirm_quasar_counts(qinfo, list(bfiles), "bfile LIKE 'G%/%.tar'")   # recount confirmed renames from RDADB
       qcnt = qinfo['qcnt']
       if qcnt > 0:
          s = 's' if qcnt > 1 else ''
@@ -1671,43 +1951,6 @@ class DsQuasar(PgCMD, PgSplit):
       dsfiles[pgrec['type']][pgrec['bid']] = {'note' : pgrec['note'], 'fcnt' : pgrec['fcnt'], 'size' : pgrec['size'],
                                               'date' : pgrec['date_modified'], 'time' : pgrec['time_modified']}
 
-   # rebuild missed file note 
-   def rebuild_file_note(self, pgrec, dsids):
-      bid = pgrec['bid']
-      fields = 'file, dsid, type, data_size, checksum'
-      bfiles = {dsids[0] : {'S' : [], 'W' : []}}
-      pgrecs = self.pgmget('sfile', 's' + fields, "bid = {} ORDER BY dsid, sfile".format(bid), self.LGEREX)
-      cnt = len(pgrecs['sfile']) if pgrecs else 0
-      for i in range(cnt):
-         dsid = pgrecs['dsid'][i]
-         if dsid not in dsids:
-            dsids.append(dsid)
-            bfiles[dsid] = {'S' : [], 'W' : []}
-         bfiles[dsid]['S'].append([pgrecs['sfile'][i], pgrecs['type'][i], pgrecs['data_size'][i], pgrecs['checksum'][i]])
-      pgrecs = self.pgmget('wfile', 'w' + fields, "bid = {} ORDER BY dsid, wfile".format(bid), self.LGEREX)
-      cnt = len(pgrecs['wfile']) if pgrecs else 0
-      for i in range(cnt):
-         dsid = pgrecs['dsid'][i]
-         if dsid not in dsids:
-            dsids.append(dsid)
-            bfiles[dsid] = {'S' : [], 'W' : []}
-         bfiles[dsid]['W'].append([pgrecs['wfile'][i], pgrecs['type'][i], pgrecs['data_size'][i], pgrecs['checksum'][i]])
-      # create note
-      note = ''
-      for dsid in bfiles:
-         for ftype in bfiles[dsid]:
-            files = bfiles[dsid][ftype]
-            if not files: continue
-            note += "<{}_{}_{}.txt>\n".format(dsid, ftype, bid)
-            note += "DS<=>{}\n{}F<:>{}T<:>SZ<:>MC<:>\n".format(dsid, ftype, ftype)
-            for flist in files:
-               note += "{}<:>{}<:>{}<:>{}<:>\n".format(flist[0], flist[1], flist[2], flist[3])
-      if note:
-         self.pgexec("UPDATE bfile SET note = '{}' WHERE bid = {}".format(note, bid), self.LGEREX)
-         pgrec['note'] = note
-         return 1
-      return 0
-
    # dump statistics for GDEX files available to create input files
    def dump_dataset_files(self, dsfiles, backflag):
       bfiles = dsfiles[backflag]
@@ -1922,7 +2165,7 @@ class DsQuasar(PgCMD, PgSplit):
             pcnt += 1
       return pcnt
 
-# main function to excecute this script
+# main function to execute this script
 def main():
    object = DsQuasar()
    object.read_parameters()

rda_python_dsquasar-3.0.6/src/rda_python_dsquasar/dsquasar.usg

@@ -0,0 +1,146 @@
+
+ Name: dsquasar - back up RDA/GDEX data archives onto the Quasar backup servers
+
+ Check the Backup flags set in dataset.backflag and/or dsgroup.backflag of RDADB
+ to gather Web and Saved files, tar them into larger (>= 2GB) backup files, and
+ back them up onto the NCAR GDEX Quasar Globus end points, for a Backup copy
+ (gdex-quasar) and/or a disaster-recovery Drdata copy (gdex-quasar-drdata).  The
+ file tarring and transferring is performed through the utility program dsarch.
+
+ Usage: dsquasar [-b] [(-a | -t DatasetIDs)] [-c ChangeDays] [-(e|E)]     \
+                 [-(B|D)] [-A ActionBits] [-l (Y|N)] [-m ProcessCount]    \
+                 [-n] [-u] [-d [HostName] [TryCount]] [-h]
+
+   Either -a or -t must be given (usage is displayed otherwise).  dsquasar runs
+   from the Quasar backup work directory; it changes there automatically.
+
+
+ DATASET SELECTION
+
+   -a     Gather files of all datasets that have a Backup flag set.
+
+   -t  DatasetIDs
+          One or more dataset IDs whose files are backed up.  The SQL wildcard
+          '%' is allowed and is expanded against the dataset table.  Ignored,
+          with a warning, when -a is also given.
+
+
+ BACKUP SCOPE
+
+   -B     Quasar Backup only (endpoint gdex-quasar) for the given datasets.
+
+   -D     Quasar Backup & Drdata (gdex-quasar and gdex-quasar-drdata).
+
+          -B and -D are mutually exclusive; the default (neither given) backs up
+          to both the Backup and the Drdata endpoints.
+
+   -c  ChangeDays
+          Used with -A 1 to re-back up files that changed within the given
+          number of days.  Must be greater than 0.
+
+
+ ACTIONS
+
+   -A  ActionBits
+          Bit flags selecting the actions to perform; the valid values are
+          1, 2, 3, 4, 6, 7, 8, 16 and it defaults to 3 (actions 1 + 2):
+
+             1  - create the dsarch input Save/Web file lists and add the
+                  backup file (bfile) placeholder records (status 'N').
+             2  - build tar files (each >= 5GB where possible) from the Saved/
+                  Web files listed in the input files (status 'N' -> 'T').
+             4  - transfer the built tar files (batched to >= 90GB per Globus
+                  transfer) to the Quasar backup end points (status 'T' -> 'A').
+             8  - check file sizes between RDADB and the Quasar servers, and
+                  reset status to 'N' when they differ.
+            16  - dump backup statistics for files backed up and files that are
+                  ready to be backed up.
+
+          Combined values: 3 = 1+2, 6 = 2+4, 7 = 1+2+4.
+
+
+ RUN CONTROL
+
+   -b     Turn on background mode; no screen display.
+
+   -d  [HostName] [TryCount]
+          Turn on delay mode to run as a batch job.  A dscheck record is created
+          and the dscheck daemon later submits the job to PBS.  On the initial
+          submit dsquasar sizes multi-processing from the detail file count and
+          reserves matching PBS cpus (walltime + select=1:ncpus=N:mem=Ngb) in the
+          dscheck qoptions; the invocation arguments are left unchanged so a
+          repeat submit with the same arguments still blocks as a duplicate.
+
+   -l  (Y|N)
+          Whether to check and lock a dataset before backing it up.  Defaults
+          to Y.
+
+   -m  ProcessCount
+          Number of concurrent child processes.  Defaults to 1.  In delay mode
+          the process count travels through the reserved PBS cpus rather than
+          the command line.
+
+   -n     Gather and show the available file counts only; no backup actions are
+          performed.
+
+   -W  MaxWorkers
+          Maximum number of concurrent batch workers for the same command.
+          Defaults to 2.  A repeat delayed submit normally blocks while the
+          batch job is running; if that job has run for more than 6 hours and
+          has still completed no more than 1% of its recorded work, it is
+          treated as stalled and an extra worker is submitted instead, up to
+          this limit.  Extra workers rely on dataset locking to stay off each
+          other's files, so this is pinned to 1 with -l N, and also for the
+          transfer action (-A 4, 6, 7), which claims tar files without locking.
+
+   -w  WorkerSlot
+          Worker slot number for this run, added automatically to an extra
+          worker's command line so it gets its own dscheck record.  Not
+          normally set by hand.  Even slots walk the datasets in descending
+          dsid order (from d999999) and odd slots in ascending order, so that a
+          pair of workers meets in the middle instead of contending for the
+          same dataset locks.
+
+   -u     Clean up (unlock) backup locks for the given dataset IDs.  Dataset IDs
+          must be provided (not valid with -a).
+
+
+ EMAIL
+
+   -e     Send an email to the specialist with backup statistics.
+
+   -E     Send an email to the specialist with detailed backup statistics.
+
+
+ MISCELLANEOUS
+
+   -h     Display this help document.
+
+
+ EXAMPLES
+
+   1. Back up (create input files and tar files for) all flagged datasets:
+
+         dsquasar -a -A 3
+
+   2. Transfer built tar files of one dataset to both Quasar end points:
+
+         dsquasar -t d123456 -A 4
+
+   3. Show the counts of files ready to back up for a dataset, without acting:
+
+         dsquasar -t d123456 -n
+
+   4. Dump backup statistics for all datasets by email:
+
+         dsquasar -a -A 16 -e
+
+   5. Queue a delayed PBS batch job to create input files and tar files for all
+      datasets in the background (as run from cron); multi-processing is sized
+      automatically from the file count:
+
+         dsquasar -a -A 3 -e -b -d PBS
+
+   6. Unlock backup locks left on datasets:
+
+         dsquasar -t d123456 -u

{rda_python_dsquasar-3.0.1 → rda_python_dsquasar-3.0.6/src/rda_python_dsquasar.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsquasar
-Version: 3.0.1
+Version: 3.0.6
 Summary: RDA Python package to backup and recover RDA data archives to and from GLOBUS Quasar backup server
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsquasar

{rda_python_dsquasar-3.0.1 → rda_python_dsquasar-3.0.6}/src/rda_python_dsquasar.egg-info/SOURCES.txt

@@ -1,9 +1,11 @@
 LICENSE
+MANIFEST.in
 README.md
 pyproject.toml
 src/rda_python_dsquasar/__init__.py
 src/rda_python_dsquasar/ds_quasar.py
 src/rda_python_dsquasar/dsquasar.py
+src/rda_python_dsquasar/dsquasar.usg
 src/rda_python_dsquasar/dstacc.py
 src/rda_python_dsquasar/taccrec.py
 src/rda_python_dsquasar/tacctar.py