rda-python-dsquasar 3.0.21__tar.gz → 3.0.23__tar.gz

{rda_python_dsquasar-3.0.21/src/rda_python_dsquasar.egg-info → rda_python_dsquasar-3.0.23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsquasar
-Version: 3.0.21
+Version: 3.0.23
 Summary: RDA Python package to backup and recover RDA data archives to and from GLOBUS Quasar backup server
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsquasar

{rda_python_dsquasar-3.0.21 → rda_python_dsquasar-3.0.23}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_dsquasar"
-version = "3.0.21"
+version = "3.0.23"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]

{rda_python_dsquasar-3.0.21 → rda_python_dsquasar-3.0.23}/src/rda_python_dsquasar/dsquasar.py

@@ -105,7 +105,7 @@ class DsQuasar(PgCMD, PgSplit):
       # dscheck work done, which is action independent because dcount counts GDEX files
       # for -A 3 but tar files for -A 2/-A 4.
       self.MAXWORKERS = 2         # default maximum concurrent workers per command (-W)
-      self.WORKGRACE = 6*3600     # 6 hours before a running job is judged on progress
+      self.WORKGRACE = 2*3600     # 2 hours before a running job is judged on progress
       self.MINWPROJ = 0.9         # projected done fraction at the walltime that counts as finishing
       self.PGBACK = {
          'workdir' : "{}/{}/quasar_backup".format(self.PGLOG['GDEXWORK'], self.PGLOG['COMMONUSER']),
@@ -123,6 +123,7 @@ class DsQuasar(PgCMD, PgSplit):
          'maxcnt' : 10,
          'dolock' : 1,
          'doemail' : 0,
+         'locflag' : None,  # -L, limit the datasets gathered to one dataset.locflag
          'starttime' : 0,   # wall-clock start of the run, for the PBS walltime guard
          'einfo' : 0,       # set once a progress report is cached into dscheck.einfo
          'eretry' : 0,      # earliest retry time after a progress report failed to cache
@@ -144,7 +145,7 @@ class DsQuasar(PgCMD, PgSplit):
       argv = sys.argv[1:]
       for arg in argv:
          if re.match(r'^-(h|-help)$', arg, re.I): self.show_usage('dsquasar')
-         ms = re.match(r'^-(a|b|c|d|e|E|l|m|n|t|u|w|A|B|D|W)$', arg)
+         ms = re.match(r'^-(a|b|c|d|e|E|l|m|n|t|u|w|A|B|D|L|W)$', arg)
          if ms:
             arg = ms.group(1)
             if  arg == 'b':
@@ -152,7 +153,7 @@ class DsQuasar(PgCMD, PgSplit):
             elif arg in self.sopts:
                self.sopts[arg] = 1
                option = None
-            elif 'AcdlmtwW'.find(arg) > -1:
+            elif 'AcdLlmtwW'.find(arg) > -1:
                option = arg
                if arg == 'd': self.bopts = []
             elif 'BD'.find(arg) > -1:
@@ -179,6 +180,10 @@ class DsQuasar(PgCMD, PgSplit):
                if not (arg == 'Y' or arg == 'N'): self.pglog(arg +": Lock Flag(-l) must be Y or N", self.LGWNEX)
                self.PGBACK['dolock'] = 1 if arg == 'Y' else 0
                option = None
+            elif option == 'L':
+               if not (arg == 'G' or arg == 'O'): self.pglog(arg +": Location Flag(-L) must be G or O", self.LGWNEX)
+               self.PGBACK['locflag'] = arg
+               option = None
             elif option == 'W':
                self.PGBACK['maxworkers'] = int(arg)
                if self.PGBACK['maxworkers'] < 1: self.pglog(arg +": Maximum Workers(-W) must be > 0", self.LGWNEX)
@@ -426,8 +431,9 @@ class DsQuasar(PgCMD, PgSplit):
          dargv = dargv[0:100]
       return self.get_dscheck("dsquasar", dargv, self.PGBACK['workdir'], self.PGLOG['CURUID'], dargx, self.LOGWRN)
 
-   # a running batch job counts as stalled once it has run longer than WORKGRACE and its
-   # progress so far projects to less than MINWPROJ of its recorded work by the walltime:
+   # a running batch job counts as stalled once it has run longer than WORKGRACE, has
+   # recorded some work done, and its progress so far projects to less than MINWPROJ of
+   # its recorded work by the walltime:
    # such a job cannot finish this attempt however long it is left alone, so an extra
    # worker is what helps it, not more retries.
    # the test is on the RATE, not on an absolute done fraction as it first was. a fixed
@@ -442,11 +448,15 @@ class DsQuasar(PgCMD, PgSplit):
    # runs this and set_walltime_deadline reads the granted walltime back in the batch job.
    # when the daemon caps it to a shorter queue limit the projection is optimistic, which
    # errs towards leaving the job alone.
+   # a job that has recorded nothing done is still in its gather phases, which read as a
+   # zero rate; it is left for the next submit rather than judged. that guard is what lets
+   # WORKGRACE stay short: an extra worker can only be added by a cron submit, every 4
+   # hours, so a 6 hour grace kept it out of the first 6 to 10 hours of every attempt.
    def worker_progress_stalled(self, pgrec):
       if not (pgrec['pid'] and pgrec['stttime']): return False
       if self.check_host_pid(pgrec['lockhost'], pgrec['pid']) <= 0: return False
       elapsed = tm() - pgrec['stttime']
-      if elapsed < self.WORKGRACE: return False
+      if elapsed < self.WORKGRACE or not pgrec['dcount']: return False
       done = pgrec['dcount']/pgrec['fcount'] if pgrec['fcount'] else 0
       return (done*self.WALLTIME/elapsed) < self.MINWPROJ
 
@@ -918,9 +928,10 @@ class DsQuasar(PgCMD, PgSplit):
    # even if PBS kills the job at the walltime. the final report follows and drops the
    # parked one if the run does finish in time. the live email buffers are saved and put
    # back, so the final report still carries everything logged before the cutoff.
-   # PBS sends no catchable warning before the kill - SIGTERM is not trapped and SIGKILL
-   # cannot be - so this parked report is the only thing standing between a killed job and
-   # a run that is never heard from. every action submitted to PBS is guarded, not just the
+   # batch_term_report covers the kill itself, but only the seconds of it: SIGKILL cannot be
+   # caught, and a process stuck in uninterruptible I/O may never run the handler. this parked
+   # report is what stands between a killed job and a run that is never heard from, so it is
+   # kept as the first line of defence. every action submitted to PBS is guarded, not just the
    # tar and transfer ones: -A 16 and the hidden -A 128 are submitted with the same 24 hour
    # walltime and used to be excluded, so they died silently.
    # the tar queue depths are only reported for the actions they describe. both depths are
@@ -1392,11 +1403,19 @@ class DsQuasar(PgCMD, PgSplit):
          ofile = self.join_paths(dsid, fname)
          info = self.check_object_file(ofile, bucket, 1, self.ERRACT)
          if not info: return False
-         lfile = "{}/{}/{}/{}".format(chome, self.PGLOG['BACKUPEP'], dsid, fname)
-         self.object_copy_local(lfile, ofile, bucket, self.ERRACT)
-         cinfo = self.check_local_file(lfile, 32, self.ERRACT)
-         if not cinfo: return False
-         if 'checksum' in cinfo: info['checksum'] = cinfo['checksum']
+         # an object still matching its record on size and time, with its checksum
+         # recorded already, has nothing to evaluate: downloading it only to md5 it
+         # again costs a transfer per file, and dsarch downloads it anyway to tar it.
+         # a directory key such as a zarr store has no checksum, so it is always fetched
+         if not (info['isfile'] and pgrec['checksum'] and info['data_size'] == pgrec['data_size'] and
+                 self.cmptime(info['date_modified'], info['time_modified'],
+                              pgrec['date_modified'], pgrec['time_modified']) == 0):
+            lfile = "{}/{}/{}/{}".format(chome, self.PGLOG['BACKUPEP'], dsid, fname)
+            # pass info on so the object is not listed a second time
+            if not self.object_copy_local(lfile, ofile, bucket, self.ERRACT, info): return False
+            cinfo = self.check_local_file(lfile, 32, self.ERRACT)
+            if not cinfo: return False
+            if 'checksum' in cinfo: info['checksum'] = cinfo['checksum']
          lname = 'OBS'
       else:
          lfile = "{}/{}/".format(chome, dsid)
@@ -1468,13 +1487,29 @@ class DsQuasar(PgCMD, PgSplit):
       fcnt = 0
       # warn per named dataset with no files; stay silent when scanning all datasets
       logact = self.LOGWRN if self.dsids else 0
+      flds = "dsid, backflag, pid, lockhost"
+      # -L splits the work by where the data sits: an Object(O) dataset has to be staged
+      # down from the object store and checksummed before it can be tarred, while a Glade(G)
+      # one is only stat'ed, so the two classes run at wildly different rates and are better
+      # off in separate runs than sharing one walltime.
+      # G is matched as "not O" rather than "= G" on purpose: dataset.locflag is only a
+      # summary of the per-file flags (set_dataset_locflag writes it only when every public
+      # web file agrees, and leaves the old value otherwise), so it can hold a stale value.
+      # as "not O" the two halves stay exhaustive and no dataset falls through both runs
+      lcnd = ''
+      if self.PGBACK['locflag'] == 'O':
+         lcnd = "locflag = 'O'"
+      elif self.PGBACK['locflag']:
+         lcnd = "locflag <> 'O'"
       if self.dsids:
          pgrecs = []
          for dsid in self.dsids:
-            pgrec = self.pgget("dataset", "dsid, backflag, pid", "dsid = '{}'".format(dsid), self.LGWNEX)
+            dcnd = "dsid = '{}'".format(dsid)
+            if lcnd: dcnd += " AND " + lcnd
+            pgrec = self.pgget("dataset", flds, dcnd, self.LGWNEX)
             if pgrec: pgrecs.append(pgrec)
       else:
-         mrecs = self.pgmget("dataset", "dsid, backflag, pid", self.dsid_order().strip(), self.LGWNEX)
+         mrecs = self.pgmget("dataset", flds, (lcnd + self.dsid_order()).strip(), self.LGWNEX)
          dcnt = len(mrecs['dsid']) if mrecs else 0
          pgrecs = [self.onerecord(mrecs, i) for i in range(dcnt)]
       for pgrec in pgrecs:
@@ -1487,7 +1522,16 @@ class DsQuasar(PgCMD, PgSplit):
          # main runs take new files (bid = 0), so it shares no record with the worker holding
          # the lock and must not skip the dataset for it
          if unlock and pgrec['pid'] and not self.PGBACK['chgdays']:
-            if self.lock_dataset(dsid, 0, self.LOGACT) < 1: continue
+            # quiet unlock: a live holder is the normal case for a second worker, and the
+            # library wording for it ("Cannot Unlock") reads like a failure. report the skip
+            # here instead, in terms of the worker that is still on the dataset
+            lstat = self.lock_dataset(dsid, 0, 0)
+            if lstat < 1:
+               # -1 is the live holder; 0 means the record went away or the db errored,
+               # which lock_dataset has already logged as an error
+               if lstat < 0:
+                  self.pglog("{}: skipped, still in use by {}/{}".format(dsid, pgrec['pid'], pgrec['lockhost']), self.LOGACT)
+               continue
          fcnt += self.get_dataset_files(dsid, dsfiles, pgrec['backflag'], logact, sizes)
       if dsfiles:
          s = 's' if fcnt > 1 else ''

{rda_python_dsquasar-3.0.21 → rda_python_dsquasar-3.0.23}/src/rda_python_dsquasar/dsquasar.usg

@@ -8,8 +8,8 @@
  file tarring and transferring is performed through the utility program dsarch.
 
  Usage: dsquasar [-b] [(-a | -t DatasetIDs)] [-c ChangeDays] [-(e|E)]     \
-                 [-(B|D)] [-A ActionBits] [-l (Y|N)] [-m ProcessCount]    \
-                 [-n] [-u] [-d [HostName] [TryCount]] [-h]
+                 [-(B|D)] [-L (G|O)] [-A ActionBits] [-l (Y|N)]           \
+                 [-m ProcessCount] [-n] [-u] [-d [HostName] [TryCount]] [-h]
 
    Either -a or -t must be given (usage is displayed otherwise).  dsquasar runs
    from the Quasar backup work directory; it changes there automatically.
@@ -24,6 +24,21 @@
           '%' is allowed and is expanded against the dataset table.  Ignored,
           with a warning, when -a is also given.
 
+   -L  (G|O)
+          Limit the run by dataset.locflag: O takes the datasets on the object
+          store, G takes everything else.  Applies to -a and to -t alike.  The
+          two classes back up at very different rates: an O dataset has every
+          file staged down from the object store and checksummed before it can
+          be tarred, while a G dataset is only stat'ed.  Running them in one job
+          lets the slow class eat the whole walltime, so give them separate
+          runs, e.g. one submit with -L G and another with -L O.  Defaults to
+          all datasets.
+
+          -L G and -L O together always cover every dataset, so a pair of runs
+          loses nothing.  The split is by dataset, though, and dataset.locflag
+          is only a summary of the per-file flags, so a dataset holding files of
+          both kinds is backed up entirely by whichever run claims it.
+
 
  BACKUP SCOPE
 
@@ -93,8 +108,9 @@
    -W  MaxWorkers
           Maximum number of concurrent batch workers for the same command.
           Defaults to 2.  A repeat delayed submit normally blocks while the
-          batch job is running; if that job has run for more than 6 hours and
-          has still completed no more than 1% of its recorded work, it is
+          batch job is running; if that job has run for more than 2 hours,
+          has recorded some work done, and at its rate so far would finish
+          less than 90% of its recorded work within the walltime, it is
           treated as stalled and an extra worker is submitted instead, up to
           this limit.  Extra workers rely on dataset locking to stay off each
           other's files, so this is pinned to 1 with -l N, and also for the

{rda_python_dsquasar-3.0.21 → rda_python_dsquasar-3.0.23}/src/rda_python_dsquasar/dstacc.py

@@ -861,11 +861,19 @@ class DsTACC(PgCMD, PgSplit):
          ofile = self.join_paths(dsid, fname)
          info = self.check_object_file(ofile, bucket, 1, self.ERRACT)
          if not info: return False
-         lfile = "{}/{}/{}/{}".format(chome, self.PGLOG['BACKUPEP'], dsid, fname)
-         self.object_copy_local(lfile, ofile, bucket, self.ERRACT)
-         cinfo = self.check_local_file(lfile, 32, self.ERRACT)
-         if not cinfo: return False
-         if 'checksum' in cinfo: info['checksum'] = cinfo['checksum']
+         # an object still matching its record on size and time, with its checksum
+         # recorded already, has nothing to evaluate: downloading it only to md5 it
+         # again costs a transfer per file, and dsarch downloads it anyway to tar it.
+         # a directory key such as a zarr store has no checksum, so it is always fetched
+         if not (info['isfile'] and pgrec['checksum'] and info['data_size'] == pgrec['data_size'] and
+                 self.cmptime(info['date_modified'], info['time_modified'],
+                              pgrec['date_modified'], pgrec['time_modified']) == 0):
+            lfile = "{}/{}/{}/{}".format(chome, self.PGLOG['BACKUPEP'], dsid, fname)
+            # pass info on so the object is not listed a second time
+            if not self.object_copy_local(lfile, ofile, bucket, self.ERRACT, info): return False
+            cinfo = self.check_local_file(lfile, 32, self.ERRACT)
+            if not cinfo: return False
+            if 'checksum' in cinfo: info['checksum'] = cinfo['checksum']
          lname = 'OBS'
       else:
          lfile = "{}/{}/".format(chome, dsid)

{rda_python_dsquasar-3.0.21 → rda_python_dsquasar-3.0.23/src/rda_python_dsquasar.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsquasar
-Version: 3.0.21
+Version: 3.0.23
 Summary: RDA Python package to backup and recover RDA data archives to and from GLOBUS Quasar backup server
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsquasar