rda-python-dsquasar 3.0.22__tar.gz → 3.0.24__tar.gz

{rda_python_dsquasar-3.0.22/src/rda_python_dsquasar.egg-info → rda_python_dsquasar-3.0.24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsquasar
-Version: 3.0.22
+Version: 3.0.24
 Summary: RDA Python package to backup and recover RDA data archives to and from GLOBUS Quasar backup server
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsquasar

{rda_python_dsquasar-3.0.22 → rda_python_dsquasar-3.0.24}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_dsquasar"
-version = "3.0.22"
+version = "3.0.24"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]

{rda_python_dsquasar-3.0.22 → rda_python_dsquasar-3.0.24}/src/rda_python_dsquasar/dsquasar.py

@@ -83,6 +83,8 @@ class DsQuasar(PgCMD, PgSplit):
       self.TARSIZE = 5*self.PGLOG['ONEGBS']   # 5GB, tarfile size limit
       self.MINSIZE = self.PGLOG['TWOGBS']     # 2GB, minimal tarfile size
       self.ONESIZE = 20*self.PGLOG['ONEGBS']  # 20GB, minimal file size to tar a single file 
+      self.MISSLOG = 10   # files named in the log per dataset when missing from storage
+      self.MISSCNT = 0    # files of the current dataset missing from storage
       self.TFCOUNT = 100          # if file count is greater, use self.MINSIZE for tar file
       self.SUBLMTS = 2000         # file count limit for a sub-group
       # tarring (TARACT) forks one child per tar file, so its parallel unit is the tar
@@ -105,7 +107,7 @@ class DsQuasar(PgCMD, PgSplit):
       # dscheck work done, which is action independent because dcount counts GDEX files
       # for -A 3 but tar files for -A 2/-A 4.
       self.MAXWORKERS = 2         # default maximum concurrent workers per command (-W)
-      self.WORKGRACE = 6*3600     # 6 hours before a running job is judged on progress
+      self.WORKGRACE = 2*3600     # 2 hours before a running job is judged on progress
       self.MINWPROJ = 0.9         # projected done fraction at the walltime that counts as finishing
       self.PGBACK = {
          'workdir' : "{}/{}/quasar_backup".format(self.PGLOG['GDEXWORK'], self.PGLOG['COMMONUSER']),
@@ -431,8 +433,9 @@ class DsQuasar(PgCMD, PgSplit):
          dargv = dargv[0:100]
       return self.get_dscheck("dsquasar", dargv, self.PGBACK['workdir'], self.PGLOG['CURUID'], dargx, self.LOGWRN)
 
-   # a running batch job counts as stalled once it has run longer than WORKGRACE and its
-   # progress so far projects to less than MINWPROJ of its recorded work by the walltime:
+   # a running batch job counts as stalled once it has run longer than WORKGRACE, has
+   # recorded some work done, and its progress so far projects to less than MINWPROJ of
+   # its recorded work by the walltime:
    # such a job cannot finish this attempt however long it is left alone, so an extra
    # worker is what helps it, not more retries.
    # the test is on the RATE, not on an absolute done fraction as it first was. a fixed
@@ -447,11 +450,15 @@ class DsQuasar(PgCMD, PgSplit):
    # runs this and set_walltime_deadline reads the granted walltime back in the batch job.
    # when the daemon caps it to a shorter queue limit the projection is optimistic, which
    # errs towards leaving the job alone.
+   # a job that has recorded nothing done is still in its gather phases, which read as a
+   # zero rate; it is left for the next submit rather than judged. that guard is what lets
+   # WORKGRACE stay short: an extra worker can only be added by a cron submit, every 4
+   # hours, so a 6 hour grace kept it out of the first 6 to 10 hours of every attempt.
    def worker_progress_stalled(self, pgrec):
       if not (pgrec['pid'] and pgrec['stttime']): return False
       if self.check_host_pid(pgrec['lockhost'], pgrec['pid']) <= 0: return False
       elapsed = tm() - pgrec['stttime']
-      if elapsed < self.WORKGRACE: return False
+      if elapsed < self.WORKGRACE or not pgrec['dcount']: return False
       done = pgrec['dcount']/pgrec['fcount'] if pgrec['fcount'] else 0
       return (done*self.WALLTIME/elapsed) < self.MINWPROJ
 
@@ -1220,6 +1227,7 @@ class DsQuasar(PgCMD, PgSplit):
       fd = None
       fcate = filetype.lower()
       tname = fcate + 'file'
+      self.MISSCNT = 0   # files missing from their storage location
       for i in range(fcnt):
          self.check_batch_deadline()
          pgrec = recs[i]
@@ -1244,6 +1252,12 @@ class DsQuasar(PgCMD, PgSplit):
          fd = None
          self.process_one_backup_file(qinfo, True, (fcnt-i) > 1)
       if fd: fd.close()
+      mcnt = self.MISSCNT
+      if mcnt:
+         # one line per dataset reaches the email; the first file names are in the log
+         s = 's' if mcnt > 1 else ''
+         ftype = 'Saved' if fcate == 's' else 'Web'
+         self.pglog("{}: {} of {} {} file{} missing from storage, not backed up".format(dsid, mcnt, fcnt, ftype, s), self.LOGACT)
 
    # initialize dataset backup dict structure
    @staticmethod
@@ -1388,7 +1402,21 @@ class DsQuasar(PgCMD, PgSplit):
          rcnt += 1
       return rcnt
 
-   # evaluate file stat on disk/object storage and modify RDADB record if not match
+   # log a GDEX file whose record waits for backup but whose data is not where its
+   # locflag says. the check functions return None for a missing file without logging
+   # it, so such a file used to be skipped silently, run after run. only the first
+   # MISSLOG files of a dataset are named, so a dataset missing thousands of files does
+   # not flood the log every run. returns None, which tells the caller the file is
+   # missing rather than failed
+   def missing_file_stat(self, dsid, cate, fname, path):
+      self.MISSCNT += 1
+      if self.MISSCNT <= self.MISSLOG:
+         self.pglog("{}-{}-{}: Missing {}".format(dsid, cate.upper(), fname, path), self.LOGWRN)
+      return None
+
+   # evaluate file stat on disk/object storage and modify RDADB record if not match.
+   # returns True to back the file up, False when it cannot be checked, and None
+   # when it is missing from its storage location
    def evaluate_file_stat(self, dsid, cate, pgrec):
       tname = cate + 'file'
       fname = pgrec[tname]
@@ -1397,18 +1425,28 @@ class DsQuasar(PgCMD, PgSplit):
          bucket = 'gdex-decsdata' if cate == 's' else self.PGLOG['OBJCTBKT']
          ofile = self.join_paths(dsid, fname)
          info = self.check_object_file(ofile, bucket, 1, self.ERRACT)
+         if info is None: return self.missing_file_stat(dsid, cate, fname, "{}:{}".format(bucket, ofile))
          if not info: return False
-         lfile = "{}/{}/{}/{}".format(chome, self.PGLOG['BACKUPEP'], dsid, fname)
-         self.object_copy_local(lfile, ofile, bucket, self.ERRACT)
-         cinfo = self.check_local_file(lfile, 32, self.ERRACT)
-         if not cinfo: return False
-         if 'checksum' in cinfo: info['checksum'] = cinfo['checksum']
+         # an object still matching its record on size and time, with its checksum
+         # recorded already, has nothing to evaluate: downloading it only to md5 it
+         # again costs a transfer per file, and dsarch downloads it anyway to tar it.
+         # a directory key such as a zarr store has no checksum, so it is always fetched
+         if not (info['isfile'] and pgrec['checksum'] and info['data_size'] == pgrec['data_size'] and
+                 self.cmptime(info['date_modified'], info['time_modified'],
+                              pgrec['date_modified'], pgrec['time_modified']) == 0):
+            lfile = "{}/{}/{}/{}".format(chome, self.PGLOG['BACKUPEP'], dsid, fname)
+            # pass info on so the object is not listed a second time
+            if not self.object_copy_local(lfile, ofile, bucket, self.ERRACT, info): return False
+            cinfo = self.check_local_file(lfile, 32, self.ERRACT)
+            if not cinfo: return False
+            if 'checksum' in cinfo: info['checksum'] = cinfo['checksum']
          lname = 'OBS'
       else:
          lfile = "{}/{}/".format(chome, dsid)
          if cate == 's': lfile += pgrec['type'] + '/'
          lfile += fname
          info = self.check_local_file(lfile, 33, self.ERRACT)
+         if info is None: return self.missing_file_stat(dsid, cate, fname, lfile)
          if not info: return False
          lname = 'DSK'
       record = {}      

{rda_python_dsquasar-3.0.22 → rda_python_dsquasar-3.0.24}/src/rda_python_dsquasar/dsquasar.usg

@@ -108,8 +108,9 @@
    -W  MaxWorkers
           Maximum number of concurrent batch workers for the same command.
           Defaults to 2.  A repeat delayed submit normally blocks while the
-          batch job is running; if that job has run for more than 6 hours and
-          has still completed no more than 1% of its recorded work, it is
+          batch job is running; if that job has run for more than 2 hours,
+          has recorded some work done, and at its rate so far would finish
+          less than 90% of its recorded work within the walltime, it is
           treated as stalled and an extra worker is submitted instead, up to
           this limit.  Extra workers rely on dataset locking to stay off each
           other's files, so this is pinned to 1 with -l N, and also for the

{rda_python_dsquasar-3.0.22 → rda_python_dsquasar-3.0.24}/src/rda_python_dsquasar/dstacc.py

@@ -79,6 +79,8 @@ class DsTACC(PgCMD, PgSplit):
       self.TARSIZE = 1024*self.PGLOG['ONEGBS']   # 1TB, tarfile size limit
       self.MINSIZE = 512*self.PGLOG['ONEGBS']    # .5TB, minimal tarfile size
       self.ONESIZE = 20*self.PGLOG['ONEGBS']  # 20GB, minimal file size to tar a single file 
+      self.MISSLOG = 10   # files named in the log per dataset when missing from storage
+      self.MISSCNT = 0    # files of the current dataset missing from storage
       self.TFCOUNT = 1000000      # if file count is greater, use self.MINSIZE for tar file
       self.PGBACK = {
          'workdir' : "{}/{}/tacc_backup".format(self.PGLOG['GDEXWORK'], self.PGLOG['COMMONUSER']),
@@ -731,6 +733,7 @@ class DsTACC(PgCMD, PgSplit):
       fd = None
       fcate = filetype.lower()
       tname = fcate + 'file'
+      self.MISSCNT = 0   # files missing from their storage location
       for i in range(fcnt):
          pgrec = recs[i]
          if not self.evaluate_file_stat(dsid, fcate, pgrec): continue
@@ -754,6 +757,12 @@ class DsTACC(PgCMD, PgSplit):
          fd = None
          self.process_one_backup_file(qinfo, True, (fcnt-i) > 1)
       if fd: fd.close()
+      mcnt = self.MISSCNT
+      if mcnt:
+         # one line per dataset reaches the email; the first file names are in the log
+         s = 's' if mcnt > 1 else ''
+         ftype = 'Saved' if fcate == 's' else 'Web'
+         self.pglog("{}: {} of {} {} file{} missing from storage, not backed up".format(dsid, mcnt, fcnt, ftype, s), self.LOGACT)
 
    # initialize dataset backup dict structure
    @staticmethod
@@ -851,7 +860,21 @@ class DsTACC(PgCMD, PgSplit):
          rcnt += 1
       return rcnt
 
-   # evaluate file stat on disk/object storage and modify RDADB record if not match
+   # log a GDEX file whose record waits for backup but whose data is not where its
+   # locflag says. the check functions return None for a missing file without logging
+   # it, so such a file used to be skipped silently, run after run. only the first
+   # MISSLOG files of a dataset are named, so a dataset missing thousands of files does
+   # not flood the log every run. returns None, which tells the caller the file is
+   # missing rather than failed
+   def missing_file_stat(self, dsid, cate, fname, path):
+      self.MISSCNT += 1
+      if self.MISSCNT <= self.MISSLOG:
+         self.pglog("{}-{}-{}: Missing {}".format(dsid, cate.upper(), fname, path), self.LOGWRN)
+      return None
+
+   # evaluate file stat on disk/object storage and modify RDADB record if not match.
+   # returns True to back the file up, False when it cannot be checked, and None
+   # when it is missing from its storage location
    def evaluate_file_stat(self, dsid, cate, pgrec):
       tname = cate + 'file'
       fname = pgrec[tname]
@@ -860,18 +883,28 @@ class DsTACC(PgCMD, PgSplit):
          bucket = 'gdex-decsdata' if cate == 's' else self.PGLOG['OBJCTBKT']
          ofile = self.join_paths(dsid, fname)
          info = self.check_object_file(ofile, bucket, 1, self.ERRACT)
+         if info is None: return self.missing_file_stat(dsid, cate, fname, "{}:{}".format(bucket, ofile))
          if not info: return False
-         lfile = "{}/{}/{}/{}".format(chome, self.PGLOG['BACKUPEP'], dsid, fname)
-         self.object_copy_local(lfile, ofile, bucket, self.ERRACT)
-         cinfo = self.check_local_file(lfile, 32, self.ERRACT)
-         if not cinfo: return False
-         if 'checksum' in cinfo: info['checksum'] = cinfo['checksum']
+         # an object still matching its record on size and time, with its checksum
+         # recorded already, has nothing to evaluate: downloading it only to md5 it
+         # again costs a transfer per file, and dsarch downloads it anyway to tar it.
+         # a directory key such as a zarr store has no checksum, so it is always fetched
+         if not (info['isfile'] and pgrec['checksum'] and info['data_size'] == pgrec['data_size'] and
+                 self.cmptime(info['date_modified'], info['time_modified'],
+                              pgrec['date_modified'], pgrec['time_modified']) == 0):
+            lfile = "{}/{}/{}/{}".format(chome, self.PGLOG['BACKUPEP'], dsid, fname)
+            # pass info on so the object is not listed a second time
+            if not self.object_copy_local(lfile, ofile, bucket, self.ERRACT, info): return False
+            cinfo = self.check_local_file(lfile, 32, self.ERRACT)
+            if not cinfo: return False
+            if 'checksum' in cinfo: info['checksum'] = cinfo['checksum']
          lname = 'OBS'
       else:
          lfile = "{}/{}/".format(chome, dsid)
          if cate == 's': lfile += pgrec['type'] + '/'
          lfile += fname
          info = self.check_local_file(lfile, 33, self.ERRACT)
+         if info is None: return self.missing_file_stat(dsid, cate, fname, lfile)
          if not info: return False
          lname = 'DSK'
       record = {}      

{rda_python_dsquasar-3.0.22 → rda_python_dsquasar-3.0.24/src/rda_python_dsquasar.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsquasar
-Version: 3.0.22
+Version: 3.0.24
 Summary: RDA Python package to backup and recover RDA data archives to and from GLOBUS Quasar backup server
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsquasar